From 3085a604300ed76d06a0011bd5555e419897b6cd Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Fri, 3 Mar 2017 13:00:22 -0800 Subject: [PATCH 001/188] HDFS-8112. Relax permission checking for EC related operations. --- .../server/namenode/FSDirErasureCodingOp.java | 74 ++++----- .../namenode/FSDirStatAndListingOp.java | 4 +- .../server/namenode/FSDirWriteFileOp.java | 12 +- .../hdfs/server/namenode/FSDirectory.java | 3 +- .../hdfs/server/namenode/FSEditLogLoader.java | 20 ++- .../hdfs/server/namenode/FSNamesystem.java | 11 +- .../hdfs/TestErasureCodingPolicies.java | 153 ++++++++++++++++-- 7 files changed, 203 insertions(+), 74 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java index 5240c7ed1b3..30a7b8bbe64 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java @@ -40,6 +40,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.WritableUtils; +import org.apache.hadoop.security.AccessControlException; import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.XATTR_ERASURECODING_POLICY; @@ -66,15 +67,15 @@ final class FSDirErasureCodingOp { * @return {@link HdfsFileStatus} * @throws IOException * @throws HadoopIllegalArgumentException if the policy is not enabled + * @throws AccessControlException if the user does not have write access */ static HdfsFileStatus setErasureCodingPolicy(final FSNamesystem fsn, final String srcArg, final String ecPolicyName, - final boolean logRetryCache) throws IOException { + final FSPermissionChecker pc, final boolean logRetryCache) + throws IOException, AccessControlException { assert fsn.hasWriteLock(); String src = srcArg; - FSPermissionChecker pc = null; - pc = fsn.getPermissionChecker(); FSDirectory fsd = fsn.getFSDirectory(); final INodesInPath iip; List xAttrs; @@ -88,6 +89,10 @@ final class FSDirErasureCodingOp { "policies."); } iip = fsd.resolvePath(pc, src, DirOp.WRITE_LINK); + // Write access is required to set erasure coding policy + if (fsd.isPermissionEnabled()) { + fsd.checkPathAccess(pc, iip, FsAction.WRITE); + } src = iip.getPath(); xAttrs = setErasureCodingPolicyXAttr(fsn, iip, ecPolicy); } finally { @@ -97,7 +102,7 @@ final class FSDirErasureCodingOp { return fsd.getAuditFileInfo(iip); } - static List setErasureCodingPolicyXAttr(final FSNamesystem fsn, + private static List setErasureCodingPolicyXAttr(final FSNamesystem fsn, final INodesInPath srcIIP, ErasureCodingPolicy ecPolicy) throws IOException { FSDirectory fsd = fsn.getFSDirectory(); assert fsd.hasWriteLock(); @@ -165,19 +170,24 @@ final class FSDirErasureCodingOp { * cache rebuilding * @return {@link HdfsFileStatus} * @throws IOException + * @throws AccessControlException if the user does not have write access */ static HdfsFileStatus unsetErasureCodingPolicy(final FSNamesystem fsn, - final String srcArg, final boolean logRetryCache) throws IOException { + final String srcArg, final FSPermissionChecker pc, + final boolean logRetryCache) throws IOException { assert fsn.hasWriteLock(); String src = srcArg; - FSPermissionChecker pc = fsn.getPermissionChecker(); FSDirectory fsd = fsn.getFSDirectory(); final INodesInPath iip; List xAttrs; fsd.writeLock(); try { iip = fsd.resolvePath(pc, src, DirOp.WRITE_LINK); + // Write access is required to unset erasure coding policy + if (fsd.isPermissionEnabled()) { + fsd.checkPathAccess(pc, iip, FsAction.WRITE); + } src = iip.getPath(); xAttrs = removeErasureCodingPolicyXAttr(fsn, iip); } finally { @@ -225,29 +235,23 @@ final class FSDirErasureCodingOp { * @return {@link ErasureCodingPolicy} * @throws IOException * @throws FileNotFoundException if the path does not exist. + * @throws AccessControlException if no read access */ static ErasureCodingPolicy getErasureCodingPolicy(final FSNamesystem fsn, - final String src) throws IOException { + final String src, FSPermissionChecker pc) + throws IOException, AccessControlException { assert fsn.hasReadLock(); - final INodesInPath iip = getINodesInPath(fsn, src); + FSDirectory fsd = fsn.getFSDirectory(); + final INodesInPath iip = fsd.resolvePath(pc, src, DirOp.READ); + if (fsn.isPermissionEnabled()) { + fsn.getFSDirectory().checkPathAccess(pc, iip, FsAction.READ); + } + if (iip.getLastINode() == null) { throw new FileNotFoundException("Path not found: " + iip.getPath()); } - return getErasureCodingPolicyForPath(fsn, iip); - } - - /** - * Check if the file or directory has an erasure coding policy. - * - * @param fsn namespace - * @param srcArg path - * @return Whether the file or directory has an erasure coding policy. - * @throws IOException - */ - static boolean hasErasureCodingPolicy(final FSNamesystem fsn, - final String srcArg) throws IOException { - return hasErasureCodingPolicy(fsn, getINodesInPath(fsn, srcArg)); + return getErasureCodingPolicyForPath(fsd, iip); } /** @@ -260,22 +264,22 @@ final class FSDirErasureCodingOp { */ static boolean hasErasureCodingPolicy(final FSNamesystem fsn, final INodesInPath iip) throws IOException { - return getErasureCodingPolicy(fsn, iip) != null; + return unprotectedGetErasureCodingPolicy(fsn, iip) != null; } /** - * Get the erasure coding policy. + * Get the erasure coding policy. This does not do any permission checking. * * @param fsn namespace * @param iip inodes in the path containing the file * @return {@link ErasureCodingPolicy} * @throws IOException */ - static ErasureCodingPolicy getErasureCodingPolicy(final FSNamesystem fsn, - final INodesInPath iip) throws IOException { + static ErasureCodingPolicy unprotectedGetErasureCodingPolicy( + final FSNamesystem fsn, final INodesInPath iip) throws IOException { assert fsn.hasReadLock(); - return getErasureCodingPolicyForPath(fsn, iip); + return getErasureCodingPolicyForPath(fsn.getFSDirectory(), iip); } /** @@ -290,21 +294,9 @@ final class FSDirErasureCodingOp { return fsn.getErasureCodingPolicyManager().getPolicies(); } - private static INodesInPath getINodesInPath(final FSNamesystem fsn, - final String srcArg) throws IOException { - final FSDirectory fsd = fsn.getFSDirectory(); - final FSPermissionChecker pc = fsn.getPermissionChecker(); - INodesInPath iip = fsd.resolvePath(pc, srcArg, DirOp.READ); - if (fsn.isPermissionEnabled()) { - fsn.getFSDirectory().checkPathAccess(pc, iip, FsAction.READ); - } - return iip; - } - - private static ErasureCodingPolicy getErasureCodingPolicyForPath(FSNamesystem fsn, - INodesInPath iip) throws IOException { + private static ErasureCodingPolicy getErasureCodingPolicyForPath( + FSDirectory fsd, INodesInPath iip) throws IOException { Preconditions.checkNotNull(iip, "INodes cannot be null"); - FSDirectory fsd = fsn.getFSDirectory(); fsd.readLock(); try { List inodes = iip.getReadOnlyINodes(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java index ba7deece812..d0431ff1929 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java @@ -172,7 +172,7 @@ class FSDirStatAndListingOp { final FileEncryptionInfo feInfo = FSDirEncryptionZoneOp.getFileEncryptionInfo(fsd, iip); final ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp. - getErasureCodingPolicy(fsd.getFSNamesystem(), iip); + unprotectedGetErasureCodingPolicy(fsd.getFSNamesystem(), iip); final LocatedBlocks blocks = bm.createLocatedBlocks( inode.getBlocks(iip.getPathSnapshotId()), fileSize, isUc, offset, @@ -413,7 +413,7 @@ class FSDirStatAndListingOp { FileEncryptionInfo feInfo = null; final ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp - .getErasureCodingPolicy(fsd.getFSNamesystem(), iip); + .unprotectedGetErasureCodingPolicy(fsd.getFSNamesystem(), iip); if (node.isFile()) { final INodeFile fileNode = node.asFile(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java index 0a4d3b0cbe8..bb920049b33 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java @@ -190,7 +190,8 @@ class FSDirWriteFileOp { blockType = pendingFile.getBlockType(); ErasureCodingPolicy ecPolicy = null; if (blockType == BlockType.STRIPED) { - ecPolicy = FSDirErasureCodingOp.getErasureCodingPolicy(fsn, src); + ecPolicy = + FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy(fsn, iip); numTargets = (short) (ecPolicy.getSchema().getNumDataUnits() + ecPolicy.getSchema().getNumParityUnits()); } else { @@ -418,7 +419,7 @@ class FSDirWriteFileOp { // check if the file has an EC policy boolean isStriped = false; ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp. - getErasureCodingPolicy(fsd.getFSNamesystem(), existing); + unprotectedGetErasureCodingPolicy(fsd.getFSNamesystem(), existing); if (ecPolicy != null) { isStriped = true; } @@ -475,8 +476,9 @@ class FSDirWriteFileOp { // associate new last block for the file final BlockInfo blockInfo; if (blockType == BlockType.STRIPED) { - ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp.getErasureCodingPolicy( - fsd.getFSNamesystem(), inodesInPath); + ErasureCodingPolicy ecPolicy = + FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy( + fsd.getFSNamesystem(), inodesInPath); short numDataUnits = (short) ecPolicy.getNumDataUnits(); short numParityUnits = (short) ecPolicy.getNumParityUnits(); short numLocations = (short) (numDataUnits + numParityUnits); @@ -529,7 +531,7 @@ class FSDirWriteFileOp { try { boolean isStriped = false; ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp. - getErasureCodingPolicy(fsd.getFSNamesystem(), existing); + unprotectedGetErasureCodingPolicy(fsd.getFSNamesystem(), existing); if (ecPolicy != null) { isStriped = true; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index b21442df5e6..9ac61496ab7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@ -957,7 +957,8 @@ public class FSDirectory implements Closeable { final short replicationFactor; if (fileINode.isStriped()) { final ErasureCodingPolicy ecPolicy = - FSDirErasureCodingOp.getErasureCodingPolicy(namesystem, iip); + FSDirErasureCodingOp + .unprotectedGetErasureCodingPolicy(namesystem, iip); final short numDataUnits = (short) ecPolicy.getNumDataUnits(); final short numParityUnits = (short) ecPolicy.getNumParityUnits(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java index e3ad0765ed7..ff36f1816f9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java @@ -415,8 +415,9 @@ public class FSEditLogLoader { // Update the salient file attributes. newFile.setAccessTime(addCloseOp.atime, Snapshot.CURRENT_STATE_ID); newFile.setModificationTime(addCloseOp.mtime, Snapshot.CURRENT_STATE_ID); - ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp.getErasureCodingPolicy( - fsDir.getFSNamesystem(), iip); + ErasureCodingPolicy ecPolicy = + FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy( + fsDir.getFSNamesystem(), iip); updateBlocks(fsDir, addCloseOp, iip, newFile, ecPolicy); break; } @@ -437,8 +438,9 @@ public class FSEditLogLoader { // Update the salient file attributes. file.setAccessTime(addCloseOp.atime, Snapshot.CURRENT_STATE_ID); file.setModificationTime(addCloseOp.mtime, Snapshot.CURRENT_STATE_ID); - ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp.getErasureCodingPolicy( - fsDir.getFSNamesystem(), iip); + ErasureCodingPolicy ecPolicy = + FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy( + fsDir.getFSNamesystem(), iip); updateBlocks(fsDir, addCloseOp, iip, file, ecPolicy); // Now close the file @@ -496,8 +498,9 @@ public class FSEditLogLoader { INodesInPath iip = fsDir.getINodesInPath(path, DirOp.READ); INodeFile oldFile = INodeFile.valueOf(iip.getLastINode(), path); // Update in-memory data structures - ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp.getErasureCodingPolicy( - fsDir.getFSNamesystem(), iip); + ErasureCodingPolicy ecPolicy = + FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy( + fsDir.getFSNamesystem(), iip); updateBlocks(fsDir, updateOp, iip, oldFile, ecPolicy); if (toAddRetryCache) { @@ -515,8 +518,9 @@ public class FSEditLogLoader { INodesInPath iip = fsDir.getINodesInPath(path, DirOp.READ); INodeFile oldFile = INodeFile.valueOf(iip.getLastINode(), path); // add the new block to the INodeFile - ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp.getErasureCodingPolicy( - fsDir.getFSNamesystem(), iip); + ErasureCodingPolicy ecPolicy = + FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy( + fsDir.getFSNamesystem(), iip); addNewBlock(addBlockOp, oldFile, ecPolicy); break; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 254ccfa492c..81c5759b573 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -6782,16 +6782,16 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, final boolean logRetryCache) throws IOException, UnresolvedLinkException, SafeModeException, AccessControlException { final String operationName = "setErasureCodingPolicy"; - checkSuperuserPrivilege(); checkOperation(OperationCategory.WRITE); HdfsFileStatus resultingStat = null; + final FSPermissionChecker pc = getPermissionChecker(); boolean success = false; writeLock(); try { checkOperation(OperationCategory.WRITE); checkNameNodeSafeMode("Cannot set erasure coding policy on " + srcArg); resultingStat = FSDirErasureCodingOp.setErasureCodingPolicy(this, - srcArg, ecPolicyName, logRetryCache); + srcArg, ecPolicyName, pc, logRetryCache); success = true; } catch (AccessControlException ace) { logAuditEvent(success, operationName, srcArg, null, @@ -6818,16 +6818,16 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, final boolean logRetryCache) throws IOException, UnresolvedLinkException, SafeModeException, AccessControlException { final String operationName = "unsetErasureCodingPolicy"; - checkSuperuserPrivilege(); checkOperation(OperationCategory.WRITE); HdfsFileStatus resultingStat = null; + final FSPermissionChecker pc = getPermissionChecker(); boolean success = false; writeLock(); try { checkOperation(OperationCategory.WRITE); checkNameNodeSafeMode("Cannot unset erasure coding policy on " + srcArg); resultingStat = FSDirErasureCodingOp.unsetErasureCodingPolicy(this, - srcArg, logRetryCache); + srcArg, pc, logRetryCache); success = true; } catch (AccessControlException ace) { logAuditEvent(success, operationName, srcArg, null, @@ -6849,10 +6849,11 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, ErasureCodingPolicy getErasureCodingPolicy(String src) throws AccessControlException, UnresolvedLinkException, IOException { checkOperation(OperationCategory.READ); + FSPermissionChecker pc = getPermissionChecker(); readLock(); try { checkOperation(OperationCategory.READ); - return FSDirErasureCodingOp.getErasureCodingPolicy(this, src); + return FSDirErasureCodingOp.getErasureCodingPolicy(this, src, pc); } finally { readUnlock("getErasureCodingPolicy"); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java index 8608626e59f..e6a822775d1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.protocol.DirectoryListing; @@ -30,12 +31,17 @@ import org.apache.hadoop.hdfs.client.HdfsAdmin; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.server.namenode.INodeFile; import org.apache.hadoop.io.erasurecode.ECSchema; +import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.security.UserGroupInformation; import org.junit.After; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.Timeout; import java.io.FileNotFoundException; import java.io.IOException; +import java.security.PrivilegedExceptionAction; import java.util.Arrays; import java.util.Collection; @@ -51,6 +57,9 @@ public class TestErasureCodingPolicies { ErasureCodingPolicyManager.getSystemDefaultPolicy(); private FSNamesystem namesystem; + @Rule + public Timeout timeout = new Timeout(60 * 1000); + @Before public void setupCluster() throws IOException { conf = new HdfsConfiguration(); @@ -74,7 +83,7 @@ public class TestErasureCodingPolicies { * for pre-existing files (with replicated blocks) in an EC dir, getListing * should report them as non-ec. */ - @Test(timeout=60000) + @Test public void testReplicatedFileUnderECDir() throws IOException { final Path dir = new Path("/ec"); final Path replicatedFile = new Path(dir, "replicatedFile"); @@ -128,7 +137,7 @@ public class TestErasureCodingPolicies { assertNotNull(files[1].getErasureCodingPolicy()); } - @Test(timeout = 60000) + @Test public void testBasicSetECPolicy() throws IOException, InterruptedException { final Path testDir = new Path("/ec"); @@ -182,7 +191,7 @@ public class TestErasureCodingPolicies { } } - @Test(timeout = 60000) + @Test public void testMoveValidity() throws IOException, InterruptedException { final Path srcECDir = new Path("/srcEC"); final Path dstECDir = new Path("/dstEC"); @@ -219,7 +228,7 @@ public class TestErasureCodingPolicies { fs.rename(nonECFile, dstECDir); } - @Test(timeout = 60000) + @Test public void testReplication() throws IOException { final Path testDir = new Path("/ec"); fs.mkdir(testDir, FsPermission.getDirDefault()); @@ -237,7 +246,7 @@ public class TestErasureCodingPolicies { assertEquals(policy, fs.getErasureCodingPolicy(fooFile)); } - @Test(timeout = 60000) + @Test public void testGetErasureCodingPolicyWithSystemDefaultECPolicy() throws Exception { String src = "/ec"; final Path ecDir = new Path(src); @@ -254,7 +263,7 @@ public class TestErasureCodingPolicies { verifyErasureCodingInfo(src + "/child1", sysDefaultECPolicy); } - @Test(timeout = 60000) + @Test public void testGetErasureCodingPolicy() throws Exception { ErasureCodingPolicy[] sysECPolicies = ErasureCodingPolicyManager.getSystemPolicies(); @@ -284,13 +293,13 @@ public class TestErasureCodingPolicies { usingECPolicy, ecPolicy); } - @Test(timeout = 60000) - public void testCreationErasureCodingZoneWithInvalidPolicy() + @Test + public void testSetInvalidPolicy() throws IOException { ECSchema rsSchema = new ECSchema("rs", 4, 2); String policyName = "RS-4-2-128k"; int cellSize = 128 * 1024; - ErasureCodingPolicy ecPolicy= + ErasureCodingPolicy ecPolicy = new ErasureCodingPolicy(policyName, rsSchema, cellSize, (byte) -1); String src = "/ecDir4-2"; final Path ecDir = new Path(src); @@ -305,7 +314,7 @@ public class TestErasureCodingPolicies { } } - @Test(timeout = 60000) + @Test public void testGetAllErasureCodingPolicies() throws Exception { ErasureCodingPolicy[] sysECPolicies = ErasureCodingPolicyManager .getSystemPolicies(); @@ -315,7 +324,7 @@ public class TestErasureCodingPolicies { allECPolicies.containsAll(Arrays.asList(sysECPolicies))); } - @Test(timeout = 60000) + @Test public void testGetErasureCodingPolicyOnANonExistentFile() throws Exception { Path path = new Path("/ecDir"); try { @@ -335,7 +344,7 @@ public class TestErasureCodingPolicies { } } - @Test(timeout = 60000) + @Test public void testMultiplePoliciesCoExist() throws Exception { ErasureCodingPolicy[] sysPolicies = ErasureCodingPolicyManager.getSystemPolicies(); @@ -355,4 +364,124 @@ public class TestErasureCodingPolicies { } } } + + @Test + public void testPermissions() throws Exception { + UserGroupInformation user = + UserGroupInformation.createUserForTesting("ecuser", + new String[]{"ecgroup"}); + FileSystem userfs = user.doAs(new PrivilegedExceptionAction() { + @Override + public FileSystem run() throws Exception { + return FileSystem.get(conf); + } + }); + HdfsAdmin useradmin = user.doAs(new PrivilegedExceptionAction() { + @Override + public HdfsAdmin run() throws Exception { + return new HdfsAdmin(userfs.getUri(), conf); + } + }); + + // Create dir and set an EC policy, create an EC file + Path ecdir = new Path("/ecdir"); + Path ecfile = new Path(ecdir, "ecfile"); + fs.setPermission(new Path("/"), new FsPermission((short)0777)); + userfs.mkdirs(ecdir); + final String ecPolicyName = + ErasureCodingPolicyManager.getSystemPolicies()[0].getName(); + useradmin.setErasureCodingPolicy(ecdir, ecPolicyName); + assertEquals("Policy not present on dir", + ecPolicyName, + useradmin.getErasureCodingPolicy(ecdir).getName()); + userfs.create(ecfile).close(); + assertEquals("Policy not present on file", + ecPolicyName, + useradmin.getErasureCodingPolicy(ecfile).getName()); + + // Unset and re-set + useradmin.unsetErasureCodingPolicy(ecdir); + useradmin.setErasureCodingPolicy(ecdir, ecPolicyName); + + // Change write permissions and make sure set and unset are denied + userfs.setPermission(ecdir, new FsPermission((short)0555)); + try { + useradmin.setErasureCodingPolicy(ecdir, ecPolicyName); + fail("Should not be able to setECPolicy without write permissions"); + } catch (AccessControlException e) { + // pass + } + try { + useradmin.unsetErasureCodingPolicy(ecdir); + fail("Should not be able to unsetECPolicy without write permissions"); + } catch (AccessControlException e) { + // pass + } + + // Change the permissions again, check that set and unset work + userfs.setPermission(ecdir, new FsPermission((short)0640)); + useradmin.unsetErasureCodingPolicy(ecdir); + useradmin.setErasureCodingPolicy(ecdir, ecPolicyName); + + // Set, unset, and get with another user should be unauthorized + UserGroupInformation nobody = + UserGroupInformation.createUserForTesting("nobody", + new String[]{"nogroup"}); + HdfsAdmin noadmin = nobody.doAs(new PrivilegedExceptionAction() { + @Override + public HdfsAdmin run() throws Exception { + return new HdfsAdmin(userfs.getUri(), conf); + } + }); + try { + noadmin.setErasureCodingPolicy(ecdir, ecPolicyName); + fail("Should not be able to setECPolicy without write permissions"); + } catch (AccessControlException e) { + // pass + } + try { + noadmin.unsetErasureCodingPolicy(ecdir); + fail("Should not be able to unsetECPolicy without write permissions"); + } catch (AccessControlException e) { + // pass + } + try { + noadmin.getErasureCodingPolicy(ecdir); + fail("Should not be able to getECPolicy without write permissions"); + } catch (AccessControlException e) { + // pass + } + + // superuser can do whatever it wants + userfs.setPermission(ecdir, new FsPermission((short)0000)); + HdfsAdmin superadmin = new HdfsAdmin(fs.getUri(), conf); + superadmin.unsetErasureCodingPolicy(ecdir); + superadmin.setErasureCodingPolicy(ecdir, ecPolicyName); + superadmin.getErasureCodingPolicy(ecdir); + + // Normal user no longer has access + try { + useradmin.getErasureCodingPolicy(ecdir); + fail("Normal user should not have access"); + } catch (AccessControlException e) { + // pass + } + try { + useradmin.setErasureCodingPolicy(ecfile, ecPolicyName); + fail("Normal user should not have access"); + } catch (AccessControlException e) { + // pass + } + try { + useradmin.unsetErasureCodingPolicy(ecfile); + fail("Normal user should not have access"); + } catch (AccessControlException e) { + // pass + } + + // Everyone has access to getting the list of EC policies + useradmin.getErasureCodingPolicies(); + noadmin.getErasureCodingPolicies(); + superadmin.getErasureCodingPolicies(); + } } From 2148b83993fd8ce73bcbc7677c57ee5028a59cd4 Mon Sep 17 00:00:00 2001 From: Ray Chiang Date: Fri, 3 Mar 2017 12:55:45 -0800 Subject: [PATCH 002/188] YARN-6218. Fix TestAMRMClient when using FairScheduler. (Miklos Szegedi via rchiang) --- .../yarn/client/api/impl/TestAMRMClient.java | 139 +++++++++++------- .../scheduler/fair/FairScheduler.java | 3 +- 2 files changed, 87 insertions(+), 55 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java index 4f73bac5a47..43c02710f8c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java @@ -64,47 +64,71 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.server.MiniYARNCluster; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; +import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.Records; import org.junit.After; -import org.junit.AfterClass; import org.junit.Assert; +import org.junit.Assume; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import org.eclipse.jetty.util.log.Log; import com.google.common.base.Supplier; +/** + * Test application master client class to resource manager. + */ +@RunWith(value = Parameterized.class) public class TestAMRMClient { - static Configuration conf = null; - static MiniYARNCluster yarnCluster = null; - static YarnClient yarnClient = null; - static List nodeReports = null; - static ApplicationAttemptId attemptId = null; - static int nodeCount = 3; + private String schedulerName = null; + private Configuration conf = null; + private MiniYARNCluster yarnCluster = null; + private YarnClient yarnClient = null; + private List nodeReports = null; + private ApplicationAttemptId attemptId = null; + private int nodeCount = 3; static final int rolling_interval_sec = 13; static final long am_expire_ms = 4000; - static Resource capability; - static Priority priority; - static Priority priority2; - static String node; - static String rack; - static String[] nodes; - static String[] racks; + private Resource capability; + private Priority priority; + private Priority priority2; + private String node; + private String rack; + private String[] nodes; + private String[] racks; private final static int DEFAULT_ITERATION = 3; - @BeforeClass - public static void setup() throws Exception { + public TestAMRMClient(String schedulerName) { + this.schedulerName = schedulerName; + } + + @Parameterized.Parameters + public static Collection data() { + List list = new ArrayList(2); + list.add(new Object[] {CapacityScheduler.class.getName()}); + list.add(new Object[] {FairScheduler.class.getName()}); + return list; + } + + @Before + public void setup() throws Exception { // start minicluster conf = new YarnConfiguration(); + conf.set(YarnConfiguration.RM_SCHEDULER, schedulerName); conf.setLong( YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, rolling_interval_sec); @@ -138,10 +162,7 @@ public class TestAMRMClient { rack = nodeReports.get(0).getRackName(); nodes = new String[]{ node }; racks = new String[]{ rack }; - } - - @Before - public void startApp() throws Exception { + // submit new app ApplicationSubmissionContext appContext = yarnClient.createApplication().getApplicationSubmissionContext(); @@ -199,13 +220,10 @@ public class TestAMRMClient { } @After - public void cancelApp() throws YarnException, IOException { + public void teardown() throws YarnException, IOException { yarnClient.killApplication(attemptId.getApplicationId()); attemptId = null; - } - - @AfterClass - public static void tearDown() { + if (yarnClient != null && yarnClient.getServiceState() == STATE.STARTED) { yarnClient.stop(); } @@ -656,8 +674,8 @@ public class TestAMRMClient { amClient.releaseAssignedContainer(container.getId()); } if(allocatedContainerCount < containersRequestedAny) { - // sleep to let NM's heartbeat to RM and trigger allocations - sleep(100); + // let NM heartbeat to RM and trigger allocations + triggerSchedulingWithNMHeartBeat(); } } @@ -678,7 +696,27 @@ public class TestAMRMClient { } } } - + + /** + * Make sure we get allocations regardless of timing issues. + */ + private void triggerSchedulingWithNMHeartBeat() { + // Simulate fair scheduler update thread + RMContext context = yarnCluster.getResourceManager().getRMContext(); + if (context.getScheduler() instanceof FairScheduler) { + FairScheduler scheduler = (FairScheduler)context.getScheduler(); + scheduler.update(); + } + // Trigger NM's heartbeat to RM and trigger allocations + for (RMNode rmNode : context.getRMNodes().values()) { + context.getScheduler().handle(new NodeUpdateSchedulerEvent(rmNode)); + } + if (context.getScheduler() instanceof FairScheduler) { + FairScheduler scheduler = (FairScheduler)context.getScheduler(); + scheduler.update(); + } + } + @Test (timeout=60000) public void testAllocationWithBlacklist() throws YarnException, IOException { AMRMClientImpl amClient = null; @@ -811,8 +849,8 @@ public class TestAMRMClient { allocatedContainerCount += allocResponse.getAllocatedContainers().size(); if(allocatedContainerCount == 0) { - // sleep to let NM's heartbeat to RM and trigger allocations - sleep(100); + // let NM heartbeat to RM and trigger allocations + triggerSchedulingWithNMHeartBeat(); } } return allocatedContainerCount; @@ -934,6 +972,8 @@ public class TestAMRMClient { @Test(timeout=60000) public void testAMRMClientWithContainerResourceChange() throws YarnException, IOException { + // Fair scheduler does not support resource change + Assume.assumeTrue(schedulerName.equals(CapacityScheduler.class.getName())); AMRMClient amClient = null; try { // start am rm client @@ -981,8 +1021,8 @@ public class TestAMRMClient { } // send allocation requests amClient.allocate(0.1f); - // sleep to let NM's heartbeat to RM and trigger allocations - sleep(150); + // let NM heartbeat to RM and trigger allocations + triggerSchedulingWithNMHeartBeat(); // get allocations AllocateResponse allocResponse = amClient.allocate(0.1f); List containers = allocResponse.getAllocatedContainers(); @@ -1012,14 +1052,14 @@ public class TestAMRMClient { if (status.getState() == ContainerState.RUNNING) { break; } - sleep(100); + sleep(10); } } } catch (YarnException e) { throw new AssertionError("Exception is not expected: " + e); } - // sleep to let NM's heartbeat to RM to confirm container launch - sleep(200); + // let NM's heartbeat to RM to confirm container launch + triggerSchedulingWithNMHeartBeat(); return containers; } @@ -1079,7 +1119,7 @@ public class TestAMRMClient { allocResponse.getUpdatedContainers(); Assert.assertEquals(1, updatedContainers.size()); // we should get increase allocation after the next NM's heartbeat to RM - sleep(150); + triggerSchedulingWithNMHeartBeat(); // get allocations allocResponse = amClient.allocate(0.1f); updatedContainers = @@ -1142,8 +1182,8 @@ public class TestAMRMClient { } if(allocatedContainerCount < containersRequestedAny) { - // sleep to let NM's heartbeat to RM and trigger allocations - sleep(100); + // let NM heartbeat to RM and trigger allocations + triggerSchedulingWithNMHeartBeat(); } } @@ -1225,8 +1265,8 @@ public class TestAMRMClient { } } if(numIterations > 0) { - // sleep to make sure NM's heartbeat - sleep(100); + // let NM heartbeat to RM and trigger allocations + triggerSchedulingWithNMHeartBeat(); } } assertEquals(0, amClient.ask.size()); @@ -1284,8 +1324,8 @@ public class TestAMRMClient { } if(allocatedContainers.size() < containersRequestedAny) { - // sleep to let NM's heartbeat to RM and trigger allocations - sleep(100); + // let NM heartbeat to RM and trigger allocations + triggerSchedulingWithNMHeartBeat(); } } @@ -1396,12 +1436,7 @@ public class TestAMRMClient { while (System.currentTimeMillis() - startTime < rolling_interval_sec * 1000) { amClient.allocate(0.1f); - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } + sleep(1000); } amClient.allocate(0.1f); @@ -1461,11 +1496,7 @@ public class TestAMRMClient { } } amClient.allocate(0.1f); - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - // DO NOTHING - } + sleep(1000); } try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 0599414085c..4f3e4f9140e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -349,7 +349,8 @@ public class FairScheduler extends * fair shares, deficits, minimum slot allocations, and amount of used and * required resources per job. */ - protected void update() { + @VisibleForTesting + public void update() { try { writeLock.lock(); From ac5ae0065a127ac150a887fa6c6f3cffd86ef733 Mon Sep 17 00:00:00 2001 From: Jing Zhao Date: Fri, 3 Mar 2017 13:31:20 -0800 Subject: [PATCH 003/188] HDFS-11476. Fix NPE in FsDatasetImpl#checkAndUpdate. Contributed by Xiaobing Zhou. --- .../datanode/fsdataset/impl/FsDatasetImpl.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index 6d00d75b53f..aff19ce97e0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -2282,12 +2282,14 @@ class FsDatasetImpl implements FsDatasetSpi { if (memBlockInfo.getGenerationStamp() != diskGS) { File memMetaFile = FsDatasetUtil.getMetaFile(diskFile, memBlockInfo.getGenerationStamp()); - if (memMetaFile.exists()) { - if (memMetaFile.compareTo(diskMetaFile) != 0) { - LOG.warn("Metadata file in memory " - + memMetaFile.getAbsolutePath() - + " does not match file found by scan " - + (diskMetaFile == null? null: diskMetaFile.getAbsolutePath())); + if (fileIoProvider.exists(vol, memMetaFile)) { + String warningPrefix = "Metadata file in memory " + + memMetaFile.getAbsolutePath() + + " does not match file found by scan "; + if (!diskMetaFileExists) { + LOG.warn(warningPrefix + "null"); + } else if (memMetaFile.compareTo(diskMetaFile) != 0) { + LOG.warn(warningPrefix + diskMetaFile.getAbsolutePath()); } } else { // Metadata file corresponding to block in memory is missing From 8db7a8c3aea3d989361f32cca5b271e9653773b6 Mon Sep 17 00:00:00 2001 From: Junping Du Date: Fri, 3 Mar 2017 14:39:31 -0800 Subject: [PATCH 004/188] YARN-6271. yarn rmadin -getGroups returns information from standby RM. Contributed by Jian He. --- .../hadoop/yarn/server/resourcemanager/AdminService.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index 9dda57e2d22..bcf7309f883 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -572,6 +572,15 @@ public class AdminService extends CompositeService implements @Override public String[] getGroupsForUser(String user) throws IOException { + String operation = "getGroupsForUser"; + UserGroupInformation ugi; + try { + ugi = checkAcls(operation); + } catch (YarnException e) { + // The interface is from hadoop-common which does not accept YarnException + throw new IOException(e); + } + checkRMStatus(ugi.getShortUserName(), operation, "get groups for user"); return UserGroupInformation.createRemoteUser(user).getGroupNames(); } From 279d187f723d01658ef8698a29263652e2a05618 Mon Sep 17 00:00:00 2001 From: Jian He Date: Fri, 3 Mar 2017 15:23:33 -0800 Subject: [PATCH 005/188] YARN-6270. WebUtils.getRMWebAppURLWithScheme() needs to honor RM HA setting. Contributed by Xuan Gong --- .../hadoop/yarn/webapp/util/WebAppUtils.java | 43 +++++++++++++++---- .../yarn/conf/TestYarnConfiguration.java | 16 +++++++ 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java index e412173b40b..64a4b2ba38f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java @@ -89,21 +89,46 @@ public class WebAppUtils { hostName + ":" + port); } } - - public static String getRMWebAppURLWithScheme(Configuration conf) { - return getHttpSchemePrefix(conf) + getRMWebAppURLWithoutScheme(conf); - } - - public static String getRMWebAppURLWithoutScheme(Configuration conf) { - if (YarnConfiguration.useHttps(conf)) { - return conf.get(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, + + public static String getRMWebAppURLWithoutScheme(Configuration conf, + boolean isHAEnabled) { + YarnConfiguration yarnConfig = new YarnConfiguration(conf); + // set RM_ID if we have not configure it. + if (isHAEnabled) { + String rmId = yarnConfig.get(YarnConfiguration.RM_HA_ID); + if (rmId == null || rmId.isEmpty()) { + List rmIds = new ArrayList<>(HAUtil.getRMHAIds(conf)); + if (rmIds != null && !rmIds.isEmpty()) { + yarnConfig.set(YarnConfiguration.RM_HA_ID, rmIds.get(0)); + } + } + } + if (YarnConfiguration.useHttps(yarnConfig)) { + if (isHAEnabled) { + return HAUtil.getConfValueForRMInstance( + YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, yarnConfig); + } + return yarnConfig.get(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS); }else { - return conf.get(YarnConfiguration.RM_WEBAPP_ADDRESS, + if (isHAEnabled) { + return HAUtil.getConfValueForRMInstance( + YarnConfiguration.RM_WEBAPP_ADDRESS, yarnConfig); + } + return yarnConfig.get(YarnConfiguration.RM_WEBAPP_ADDRESS, YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS); } } + public static String getRMWebAppURLWithScheme(Configuration conf) { + return getHttpSchemePrefix(conf) + getRMWebAppURLWithoutScheme( + conf, HAUtil.isHAEnabled(conf)); + } + + public static String getRMWebAppURLWithoutScheme(Configuration conf) { + return getRMWebAppURLWithoutScheme(conf, false); + } + public static List getProxyHostsAndPortsForAmFilter( Configuration conf) { List addrs = new ArrayList(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java index 2e76865ec7f..738942300a0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java @@ -41,6 +41,22 @@ public class TestYarnConfiguration { // specifically add slashes and Jetty doesn't handle double slashes. Assert.assertNotSame("RM Web Url is not correct", "http://0.0.0.0:8088", rmWebUrl); + + // test it in HA scenario + conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); + conf.set(YarnConfiguration.RM_HA_IDS, "rm1, rm2"); + conf.set("yarn.resourcemanager.webapp.address.rm1", "10.10.10.10:18088"); + conf.set("yarn.resourcemanager.webapp.address.rm2", "20.20.20.20:28088"); + String rmWebUrlinHA = WebAppUtils.getRMWebAppURLWithScheme(conf); + Assert.assertEquals("http://10.10.10.10:18088", rmWebUrlinHA); + + YarnConfiguration conf2 = new YarnConfiguration(); + conf2.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); + conf2.set(YarnConfiguration.RM_HA_IDS, "rm1, rm2"); + conf2.set("yarn.resourcemanager.hostname.rm1", "30.30.30.30"); + conf2.set("yarn.resourcemanager.hostname.rm2", "40.40.40.40"); + String rmWebUrlinHA2 = WebAppUtils.getRMWebAppURLWithScheme(conf2); + Assert.assertEquals("http://30.30.30.30:8088", rmWebUrlinHA2); } @Test From 6b7cd62b8cf12616b13142f2eb2cfc2f25796f0f Mon Sep 17 00:00:00 2001 From: Mingliang Liu Date: Fri, 3 Mar 2017 17:06:03 -0800 Subject: [PATCH 006/188] HADOOP-13930. Azure: Add Authorization support to WASB. Contributed by Sivaguru Sankaridurg and Dushyanth --- .../src/main/resources/core-default.xml | 10 + .../conf/TestCommonConfigurationFields.java | 2 + .../fs/azure/AzureNativeFileSystemStore.java | 4 +- .../fs/azure/NativeAzureFileSystem.java | 155 +++++++++- .../fs/azure/RemoteSASKeyGeneratorImpl.java | 185 ++++++++---- .../fs/azure/RemoteWasbAuthorizerImpl.java | 247 ++++++++++++++++ .../fs/azure/SecureStorageInterfaceImpl.java | 6 +- .../fs/azure/WasbAuthorizationException.java | 40 +++ .../fs/azure/WasbAuthorizationOperations.java | 44 +++ .../fs/azure/WasbAuthorizerInterface.java | 47 +++ .../hadoop/fs/azure/security/Constants.java | 54 ++++ .../WasbDelegationTokenIdentifier.java | 48 +++ .../fs/azure/security/WasbTokenRenewer.java | 124 ++++++++ .../hadoop/fs/azure/security/package.html | 28 ++ ...ache.hadoop.security.token.TokenIdentifier | 16 + ....apache.hadoop.security.token.TokenRenewer | 16 + .../hadoop-azure/src/site/markdown/index.md | 34 +++ .../fs/azure/MockWasbAuthorizerImpl.java | 102 +++++++ ...estNativeAzureFileSystemAuthorization.java | 277 ++++++++++++++++++ 19 files changed, 1374 insertions(+), 65 deletions(-) create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationException.java create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationOperations.java create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizerInterface.java create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/Constants.java create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbDelegationTokenIdentifier.java create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbTokenRenewer.java create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/package.html create mode 100644 hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier create mode 100644 hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 35be56bf81e..52b58eddb1e 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -1292,6 +1292,16 @@ to specify the time (such as 2s, 2m, 1h, etc.). + + fs.azure.authorization + false + + Config flag to enable authorization support in WASB. Setting it to "true" enables + authorization support to WASB. Currently WASB authorization requires a remote service + to provide authorization that needs to be specified via fs.azure.authorization.remote.service.url + configuration + + diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java index 966a8ac0d08..7410d297a09 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java @@ -181,6 +181,8 @@ public class TestCommonConfigurationFields extends TestConfigurationFieldsBase { xmlPropsToSkipCompare.add("io.compression.codec.bzip2.library"); // - org.apache.hadoop.io.SequenceFile xmlPropsToSkipCompare.add("io.seqfile.local.dir"); + // - org.apache.hadoop.fs.azure.NativeAzureFileSystem + xmlPropsToSkipCompare.add("fs.azure.authorization"); } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java index 07c389ce0c2..9d7ac80cdfd 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java @@ -303,7 +303,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { private boolean useSecureMode = false; private boolean useLocalSasKeyMode = false; - private String delegationToken; + /** * A test hook interface that can modify the operation context we use for * Azure Storage operations, e.g. to inject errors. @@ -478,7 +478,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { this.storageInteractionLayer = new StorageInterfaceImpl(); } else { this.storageInteractionLayer = new SecureStorageInterfaceImpl( - useLocalSasKeyMode, conf, delegationToken); + useLocalSasKeyMode, conf); } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java index b742e533712..0dfefaf196c 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java @@ -25,9 +25,12 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.net.InetAddress; import java.net.URI; import java.net.URISyntaxException; +import java.net.URL; import java.nio.charset.Charset; +import java.security.PrivilegedExceptionAction; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; @@ -61,10 +64,15 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation; import org.apache.hadoop.fs.azure.metrics.AzureFileSystemMetricsSystem; +import org.apache.hadoop.fs.azure.security.Constants; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator; +import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticator; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.Time; import org.slf4j.Logger; @@ -1096,7 +1104,39 @@ public class NativeAzureFileSystem extends FileSystem { // A counter to create unique (within-process) names for my metrics sources. private static AtomicInteger metricsSourceNameCounter = new AtomicInteger(); private boolean appendSupportEnabled = false; - + private DelegationTokenAuthenticatedURL authURL; + private DelegationTokenAuthenticatedURL.Token authToken = new DelegationTokenAuthenticatedURL.Token(); + private String credServiceUrl; + + /** + * Configuration key to enable authorization support in WASB. + */ + public static final String KEY_AZURE_AUTHORIZATION = + "fs.azure.authorization"; + + /** + * Default value for the authorization support in WASB. + */ + private static final boolean DEFAULT_AZURE_AUTHORIZATION = false; + + /** + * Flag controlling authorization support in WASB. + */ + private boolean azureAuthorization = false; + + /** + * Flag controlling Kerberos support in WASB. + */ + private boolean kerberosSupportEnabled = false; + + /** + * Authorizer to use when authorization support is enabled in + * WASB. + */ + private WasbAuthorizerInterface authorizer = null; + + private String delegationToken = null; + public NativeAzureFileSystem() { // set store in initialize() } @@ -1227,6 +1267,31 @@ public class NativeAzureFileSystem extends FileSystem { // Initialize thread counts from user configuration deleteThreadCount = conf.getInt(AZURE_DELETE_THREADS, DEFAULT_AZURE_DELETE_THREADS); renameThreadCount = conf.getInt(AZURE_RENAME_THREADS, DEFAULT_AZURE_RENAME_THREADS); + + this.azureAuthorization = conf.getBoolean(KEY_AZURE_AUTHORIZATION, + DEFAULT_AZURE_AUTHORIZATION); + this.kerberosSupportEnabled = conf.getBoolean( + Constants.AZURE_KERBEROS_SUPPORT_PROPERTY_NAME, false); + + if (this.azureAuthorization) { + + this.authorizer = + new RemoteWasbAuthorizerImpl(); + authorizer.init(conf); + } + if (UserGroupInformation.isSecurityEnabled() && kerberosSupportEnabled) { + DelegationTokenAuthenticator authenticator = new KerberosDelegationTokenAuthenticator(); + authURL = new DelegationTokenAuthenticatedURL(authenticator); + credServiceUrl = conf.get(Constants.KEY_CRED_SERVICE_URL, String + .format("http://%s:%s", + InetAddress.getLocalHost().getCanonicalHostName(), + Constants.DEFAULT_CRED_SERVICE_PORT)); + } + } + + @VisibleForTesting + public void updateWasbAuthorizer(WasbAuthorizerInterface authorizer) { + this.authorizer = authorizer; } private NativeFileSystemStore createDefaultStore(Configuration conf) { @@ -1340,6 +1405,15 @@ public class NativeAzureFileSystem extends FileSystem { return store; } + private void performAuthCheck(String path, String accessType, + String operation) throws WasbAuthorizationException, IOException { + + if (azureAuthorization && !this.authorizer.authorize(path, accessType)) { + throw new WasbAuthorizationException(operation + + " operation for Path : " + path + " not allowed"); + } + } + /** * Gets the metrics source for this file system. * This is mainly here for unit testing purposes. @@ -1362,6 +1436,10 @@ public class NativeAzureFileSystem extends FileSystem { LOG.debug("Opening file: {} for append", f); Path absolutePath = makeAbsolute(f); + + performAuthCheck(absolutePath.toString(), + WasbAuthorizationOperations.WRITE.toString(), "append"); + String key = pathToKey(absolutePath); FileMetadata meta = null; try { @@ -1562,6 +1640,10 @@ public class NativeAzureFileSystem extends FileSystem { } Path absolutePath = makeAbsolute(f); + + performAuthCheck(absolutePath.toString(), + WasbAuthorizationOperations.WRITE.toString(), "create"); + String key = pathToKey(absolutePath); FileMetadata existingMetadata = store.retrieveMetadata(key); @@ -1684,6 +1766,10 @@ public class NativeAzureFileSystem extends FileSystem { LOG.debug("Deleting file: {}", f.toString()); Path absolutePath = makeAbsolute(f); + + performAuthCheck(absolutePath.toString(), + WasbAuthorizationOperations.EXECUTE.toString(), "delete"); + String key = pathToKey(absolutePath); // Capture the metadata for the path. @@ -1954,6 +2040,10 @@ public class NativeAzureFileSystem extends FileSystem { // Capture the absolute path and the path to key. Path absolutePath = makeAbsolute(f); + + performAuthCheck(absolutePath.toString(), + WasbAuthorizationOperations.EXECUTE.toString(), "getFileStatus"); + String key = pathToKey(absolutePath); if (key.length() == 0) { // root always exists return newDirectory(null, absolutePath); @@ -2052,6 +2142,10 @@ public class NativeAzureFileSystem extends FileSystem { LOG.debug("Listing status for {}", f.toString()); Path absolutePath = makeAbsolute(f); + + performAuthCheck(absolutePath.toString(), + WasbAuthorizationOperations.EXECUTE.toString(), "list"); + String key = pathToKey(absolutePath); Set status = new TreeSet(); FileMetadata meta = null; @@ -2274,6 +2368,10 @@ public class NativeAzureFileSystem extends FileSystem { } Path absolutePath = makeAbsolute(f); + + performAuthCheck(absolutePath.toString(), + WasbAuthorizationOperations.EXECUTE.toString(), "mkdirs"); + PermissionStatus permissionStatus = null; if(noUmask) { // ensure owner still has wx permissions at the minimum @@ -2327,6 +2425,10 @@ public class NativeAzureFileSystem extends FileSystem { LOG.debug("Opening file: {}", f.toString()); Path absolutePath = makeAbsolute(f); + + performAuthCheck(absolutePath.toString(), + WasbAuthorizationOperations.READ.toString(), "read"); + String key = pathToKey(absolutePath); FileMetadata meta = null; try { @@ -2383,7 +2485,12 @@ public class NativeAzureFileSystem extends FileSystem { + " through WASB that has colons in the name"); } - String srcKey = pathToKey(makeAbsolute(src)); + Path absolutePath = makeAbsolute(src); + + performAuthCheck(absolutePath.toString(), + WasbAuthorizationOperations.EXECUTE.toString(), "rename"); + + String srcKey = pathToKey(absolutePath); if (srcKey.length() == 0) { // Cannot rename root of file system @@ -2685,6 +2792,10 @@ public class NativeAzureFileSystem extends FileSystem { @Override public void setPermission(Path p, FsPermission permission) throws FileNotFoundException, IOException { Path absolutePath = makeAbsolute(p); + + performAuthCheck(absolutePath.toString(), + WasbAuthorizationOperations.EXECUTE.toString(), "setPermission"); + String key = pathToKey(absolutePath); FileMetadata metadata = null; try { @@ -2723,6 +2834,10 @@ public class NativeAzureFileSystem extends FileSystem { public void setOwner(Path p, String username, String groupname) throws IOException { Path absolutePath = makeAbsolute(p); + + performAuthCheck(absolutePath.toString(), + WasbAuthorizationOperations.EXECUTE.toString(), "setOwner"); + String key = pathToKey(absolutePath); FileMetadata metadata = null; @@ -2785,6 +2900,42 @@ public class NativeAzureFileSystem extends FileSystem { isClosed = true; } + @Override + public Token getDelegationToken(final String renewer) throws IOException { + if(kerberosSupportEnabled) { + try { + final UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + UserGroupInformation connectUgi = ugi.getRealUser(); + final UserGroupInformation proxyUser = connectUgi; + if (connectUgi == null) { + connectUgi = ugi; + } + if(!connectUgi.hasKerberosCredentials()){ + connectUgi = UserGroupInformation.getLoginUser(); + } + connectUgi.checkTGTAndReloginFromKeytab(); + return connectUgi.doAs(new PrivilegedExceptionAction>() { + @Override + public Token run() throws Exception { + return authURL.getDelegationToken(new URL(credServiceUrl + + Constants.DEFAULT_DELEGATION_TOKEN_MANAGER_ENDPOINT), + authToken, renewer, (proxyUser != null)? ugi.getShortUserName(): null); + } + }); + } catch (Exception ex) { + LOG.error("Error in fetching the delegation token from remote service", + ex); + if (ex instanceof IOException) { + throw (IOException) ex; + } else { + throw new IOException(ex); + } + } + } else { + return super.getDelegationToken(renewer); + } + } + /** * A handler that defines what to do with blobs whose upload was * interrupted. diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteSASKeyGeneratorImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteSASKeyGeneratorImpl.java index 404419d7422..43672b27a66 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteSASKeyGeneratorImpl.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteSASKeyGeneratorImpl.java @@ -19,10 +19,23 @@ package org.apache.hadoop.fs.azure; import java.io.IOException; +import java.net.InetAddress; import java.net.URI; import java.net.URISyntaxException; +import java.net.UnknownHostException; +import java.security.PrivilegedExceptionAction; +import java.util.Iterator; +import org.apache.commons.lang.Validate; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.azure.security.Constants; +import org.apache.hadoop.fs.azure.security.WasbDelegationTokenIdentifier; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.client.AuthenticatedURL; +import org.apache.hadoop.security.authentication.client.Authenticator; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticator; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.utils.URIBuilder; import org.slf4j.Logger; @@ -43,12 +56,6 @@ public class RemoteSASKeyGeneratorImpl extends SASKeyGeneratorImpl { public static final Logger LOG = LoggerFactory.getLogger(AzureNativeFileSystemStore.class); - /** - * Configuration parameter name expected in the Configuration - * object to provide the url of the remote service {@value} - */ - private static final String KEY_CRED_SERVICE_URL = - "fs.azure.cred.service.url"; /** * Container SAS Key generation OP name. {@value} @@ -82,7 +89,7 @@ public class RemoteSASKeyGeneratorImpl extends SASKeyGeneratorImpl { * Query parameter name for user info {@value} */ private static final String DELEGATION_TOKEN_QUERY_PARAM_NAME = - "delegation_token"; + "delegation"; /** * Query parameter name for the relative path inside the storage @@ -94,23 +101,39 @@ public class RemoteSASKeyGeneratorImpl extends SASKeyGeneratorImpl { private String delegationToken = ""; private String credServiceUrl = ""; private WasbRemoteCallHelper remoteCallHelper = null; + private boolean isSecurityEnabled; + private boolean isKerberosSupportEnabled; public RemoteSASKeyGeneratorImpl(Configuration conf) { super(conf); } - public boolean initialize(Configuration conf, String delegationToken) { + public boolean initialize(Configuration conf) { LOG.debug("Initializing RemoteSASKeyGeneratorImpl instance"); - credServiceUrl = conf.get(KEY_CRED_SERVICE_URL); - - if (delegationToken == null || delegationToken.isEmpty()) { - LOG.error("Delegation Token not provided for initialization" - + " of RemoteSASKeyGenerator"); - return false; + Iterator> tokenIterator = null; + try { + tokenIterator = UserGroupInformation.getCurrentUser().getCredentials() + .getAllTokens().iterator(); + while (tokenIterator.hasNext()) { + Token iteratedToken = tokenIterator.next(); + if (iteratedToken.getKind().equals(WasbDelegationTokenIdentifier.TOKEN_KIND)) { + delegationToken = iteratedToken.encodeToUrlString(); + } + } + } catch (IOException e) { + LOG.error("Error in fetching the WASB delegation token"); } - this.delegationToken = delegationToken; + try { + credServiceUrl = conf.get(Constants.KEY_CRED_SERVICE_URL, String + .format("http://%s:%s", + InetAddress.getLocalHost().getCanonicalHostName(), + Constants.DEFAULT_CRED_SERVICE_PORT)); + } catch (UnknownHostException e) { + LOG.error("Invalid CredService Url, configure it correctly."); + return false; + } if (credServiceUrl == null || credServiceUrl.isEmpty()) { LOG.error("CredService Url not found in configuration to initialize" @@ -119,16 +142,17 @@ public class RemoteSASKeyGeneratorImpl extends SASKeyGeneratorImpl { } remoteCallHelper = new WasbRemoteCallHelper(); - LOG.debug("Initialization of RemoteSASKeyGenerator instance successfull"); + this.isSecurityEnabled = UserGroupInformation.isSecurityEnabled(); + this.isKerberosSupportEnabled = conf.getBoolean( + Constants.AZURE_KERBEROS_SUPPORT_PROPERTY_NAME, false); + LOG.debug("Initialization of RemoteSASKeyGenerator instance successful"); return true; } @Override public URI getContainerSASUri(String storageAccount, String container) throws SASKeyGenerationException { - try { - LOG.debug("Generating Container SAS Key for Container {} " + "inside Storage Account {} ", container, storageAccount); URIBuilder uriBuilder = new URIBuilder(credServiceUrl); @@ -139,84 +163,131 @@ public class RemoteSASKeyGeneratorImpl extends SASKeyGeneratorImpl { container); uriBuilder.addParameter(SAS_EXPIRY_QUERY_PARAM_NAME, "" + getSasKeyExpiryPeriod()); - uriBuilder.addParameter(DELEGATION_TOKEN_QUERY_PARAM_NAME, - this.delegationToken); - - RemoteSASKeyGenerationResponse sasKeyResponse = - makeRemoteRequest(uriBuilder.build()); - - if (sasKeyResponse == null) { - throw new SASKeyGenerationException("RemoteSASKeyGenerationResponse" - + " object null from remote call"); - } else if (sasKeyResponse.getResponseCode() - == REMOTE_CALL_SUCCESS_CODE) { - return new URI(sasKeyResponse.getSasKey()); - } else { - throw new SASKeyGenerationException("Remote Service encountered error" - + " in SAS Key generation : " - + sasKeyResponse.getResponseMessage()); + if (isSecurityEnabled && (delegationToken != null && !delegationToken + .isEmpty())) { + uriBuilder.addParameter(DELEGATION_TOKEN_QUERY_PARAM_NAME, + this.delegationToken); } + + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + UserGroupInformation connectUgi = ugi.getRealUser(); + if (connectUgi == null) { + connectUgi = ugi; + } else { + uriBuilder.addParameter(Constants.DOAS_PARAM, ugi.getShortUserName()); + } + + if(isSecurityEnabled && !connectUgi.hasKerberosCredentials()){ + connectUgi = UserGroupInformation.getLoginUser(); + } + return getSASKey(uriBuilder.build(), connectUgi); } catch (URISyntaxException uriSyntaxEx) { throw new SASKeyGenerationException("Encountered URISyntaxException " + "while building the HttpGetRequest to remote cred service", uriSyntaxEx); + } catch (IOException e) { + throw new SASKeyGenerationException("Encountered IOException" + + " while building the HttpGetRequest to remote service", e); } } @Override public URI getRelativeBlobSASUri(String storageAccount, String container, String relativePath) throws SASKeyGenerationException { - try { - LOG.debug("Generating RelativePath SAS Key for relativePath {} inside" + " Container {} inside Storage Account {} ", relativePath, container, storageAccount); URIBuilder uriBuilder = new URIBuilder(credServiceUrl); uriBuilder.setPath("/" + BLOB_SAS_OP); - uriBuilder.addParameter(STORAGE_ACCOUNT_QUERY_PARAM_NAME, - storageAccount); - uriBuilder.addParameter(CONTAINER_QUERY_PARAM_NAME, - container); + uriBuilder.addParameter(STORAGE_ACCOUNT_QUERY_PARAM_NAME, storageAccount); + uriBuilder.addParameter(CONTAINER_QUERY_PARAM_NAME, container); uriBuilder.addParameter(RELATIVE_PATH_QUERY_PARAM_NAME, relativePath); uriBuilder.addParameter(SAS_EXPIRY_QUERY_PARAM_NAME, "" + getSasKeyExpiryPeriod()); - uriBuilder.addParameter(DELEGATION_TOKEN_QUERY_PARAM_NAME, - this.delegationToken); - RemoteSASKeyGenerationResponse sasKeyResponse = - makeRemoteRequest(uriBuilder.build()); - - if (sasKeyResponse == null) { - throw new SASKeyGenerationException("RemoteSASKeyGenerationResponse" - + " object null from remote call"); - } else if (sasKeyResponse.getResponseCode() - == REMOTE_CALL_SUCCESS_CODE) { - return new URI(sasKeyResponse.getSasKey()); - } else { - throw new SASKeyGenerationException("Remote Service encountered error" - + " in SAS Key generation : " - + sasKeyResponse.getResponseMessage()); + if (isSecurityEnabled && (delegationToken != null && !delegationToken + .isEmpty())) { + uriBuilder.addParameter(DELEGATION_TOKEN_QUERY_PARAM_NAME, + this.delegationToken); } + + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + UserGroupInformation connectUgi = ugi.getRealUser(); + if (connectUgi == null) { + connectUgi = ugi; + } else{ + uriBuilder.addParameter(Constants.DOAS_PARAM, ugi.getShortUserName()); + } + + if(isSecurityEnabled && !connectUgi.hasKerberosCredentials()){ + connectUgi = UserGroupInformation.getLoginUser(); + } + return getSASKey(uriBuilder.build(), connectUgi); } catch (URISyntaxException uriSyntaxEx) { throw new SASKeyGenerationException("Encountered URISyntaxException" + " while building the HttpGetRequest to " + " remote service", uriSyntaxEx); + } catch (IOException e) { + throw new SASKeyGenerationException("Encountered IOException" + + " while building the HttpGetRequest to remote service", e); + } + } + + private URI getSASKey(final URI uri, UserGroupInformation connectUgi) + throws URISyntaxException, SASKeyGenerationException { + RemoteSASKeyGenerationResponse sasKeyResponse = null; + try { + connectUgi.checkTGTAndReloginFromKeytab(); + sasKeyResponse = connectUgi.doAs(new PrivilegedExceptionAction() { + @Override + public RemoteSASKeyGenerationResponse run() throws Exception { + AuthenticatedURL.Token token = null; + if (isKerberosSupportEnabled && UserGroupInformation.isSecurityEnabled() && ( + delegationToken == null || delegationToken.isEmpty())) { + token = new AuthenticatedURL.Token(); + final Authenticator kerberosAuthenticator = new KerberosDelegationTokenAuthenticator(); + kerberosAuthenticator.authenticate(uri.toURL(), token); + Validate.isTrue(token.isSet(), + "Authenticated Token is NOT present. The request cannot proceed."); + } + return makeRemoteRequest(uri, (token != null ? token.toString() : null)); + } + }); + } catch (InterruptedException e) { + LOG.error("Error fetching the SAS Key from Remote Service", e); + } catch (IOException e) { + LOG.error("Error fetching the SAS Key from Remote Service", e); + } + + if (sasKeyResponse == null) { + throw new SASKeyGenerationException( + "RemoteSASKeyGenerationResponse" + " object null from remote call"); + } else if (sasKeyResponse.getResponseCode() == REMOTE_CALL_SUCCESS_CODE) { + return new URI(sasKeyResponse.getSasKey()); + } else { + throw new SASKeyGenerationException("Remote Service encountered error" + + " in SAS Key generation : " + sasKeyResponse.getResponseMessage()); } } /** * Helper method to make a remote request. * @param uri - Uri to use for the remote request + * @param token - hadoop.auth token for the remote request * @return RemoteSASKeyGenerationResponse */ - private RemoteSASKeyGenerationResponse makeRemoteRequest(URI uri) + private RemoteSASKeyGenerationResponse makeRemoteRequest(URI uri, String token) throws SASKeyGenerationException { try { + HttpGet httpGet = new HttpGet(uri); + if(token != null){ + httpGet.setHeader("Cookie", AuthenticatedURL.AUTH_COOKIE + "=" + token); + } String responseBody = - remoteCallHelper.makeRemoteGetRequest(new HttpGet(uri)); + remoteCallHelper.makeRemoteGetRequest(httpGet); ObjectMapper objectMapper = new ObjectMapper(); return objectMapper.readValue(responseBody, diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java new file mode 100644 index 00000000000..bb1093d3ff2 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java @@ -0,0 +1,247 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.URISyntaxException; +import java.security.PrivilegedExceptionAction; +import java.util.Iterator; + +import org.apache.commons.lang.Validate; +import org.apache.hadoop.conf.Configuration; + +import org.apache.hadoop.fs.azure.security.Constants; +import org.apache.hadoop.fs.azure.security.WasbDelegationTokenIdentifier; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.client.AuthenticatedURL; +import org.apache.hadoop.security.authentication.client.Authenticator; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticator; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.utils.URIBuilder; + +import com.fasterxml.jackson.core.JsonParseException; +import org.codehaus.jackson.map.JsonMappingException; +import org.codehaus.jackson.map.ObjectMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.hadoop.fs.azure.WasbRemoteCallHelper.REMOTE_CALL_SUCCESS_CODE; + +/** + * Class implementing WasbAuthorizerInterface using a remote + * service that implements the authorization operation. This + * class expects the url of the remote service to be passed + * via config. + */ +public class RemoteWasbAuthorizerImpl implements WasbAuthorizerInterface { + + public static final Logger LOG = + LoggerFactory.getLogger(RemoteWasbAuthorizerImpl.class); + private String remoteAuthorizerServiceUrl = ""; + + /** + * Configuration parameter name expected in the Configuration object to + * provide the url of the remote service. {@value} + */ + public static final String KEY_REMOTE_AUTH_SERVICE_URL = + "fs.azure.authorization.remote.service.url"; + + /** + * Authorization operation OP name in the remote service {@value} + */ + private static final String CHECK_AUTHORIZATION_OP = + "CHECK_AUTHORIZATION"; + + /** + * Query parameter specifying the access operation type. {@value} + */ + private static final String ACCESS_OPERATION_QUERY_PARAM_NAME = + "operation_type"; + + /** + * Query parameter specifying the wasb absolute path. {@value} + */ + private static final String WASB_ABSOLUTE_PATH_QUERY_PARAM_NAME = + "wasb_absolute_path"; + + /** + * Query parameter name for user info {@value} + */ + private static final String DELEGATION_TOKEN_QUERY_PARAM_NAME = + "delegation"; + + private WasbRemoteCallHelper remoteCallHelper = null; + private String delegationToken; + private boolean isSecurityEnabled; + private boolean isKerberosSupportEnabled; + + @Override + public void init(Configuration conf) + throws WasbAuthorizationException, IOException { + LOG.debug("Initializing RemoteWasbAuthorizerImpl instance"); + delegationToken = UserGroupInformation.getCurrentUser().getCredentials().getToken(WasbDelegationTokenIdentifier.TOKEN_KIND).encodeToUrlString(); + + remoteAuthorizerServiceUrl = conf.get(KEY_REMOTE_AUTH_SERVICE_URL, String + .format("http://%s:%s", + InetAddress.getLocalHost().getCanonicalHostName(), + Constants.DEFAULT_CRED_SERVICE_PORT)); + + if (remoteAuthorizerServiceUrl == null + || remoteAuthorizerServiceUrl.isEmpty()) { + throw new WasbAuthorizationException( + "fs.azure.authorization.remote.service.url config not set" + + " in configuration."); + } + + this.remoteCallHelper = new WasbRemoteCallHelper(); + this.isSecurityEnabled = UserGroupInformation.isSecurityEnabled(); + this.isKerberosSupportEnabled = conf.getBoolean( + Constants.AZURE_KERBEROS_SUPPORT_PROPERTY_NAME, false); + } + + @Override + public boolean authorize(String wasbAbsolutePath, String accessType) + throws WasbAuthorizationException, IOException { + try { + final URIBuilder uriBuilder = new URIBuilder(remoteAuthorizerServiceUrl); + uriBuilder.setPath("/" + CHECK_AUTHORIZATION_OP); + uriBuilder.addParameter(WASB_ABSOLUTE_PATH_QUERY_PARAM_NAME, + wasbAbsolutePath); + uriBuilder.addParameter(ACCESS_OPERATION_QUERY_PARAM_NAME, + accessType); + if (isSecurityEnabled && (delegationToken != null && !delegationToken + .isEmpty())) { + uriBuilder + .addParameter(DELEGATION_TOKEN_QUERY_PARAM_NAME, delegationToken); + } + String responseBody = null; + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + UserGroupInformation connectUgi = ugi.getRealUser(); + if (connectUgi == null) { + connectUgi = ugi; + } else{ + uriBuilder.addParameter(Constants.DOAS_PARAM, ugi.getShortUserName()); + } + if(isSecurityEnabled && !connectUgi.hasKerberosCredentials()){ + connectUgi = UserGroupInformation.getLoginUser(); + } + connectUgi.checkTGTAndReloginFromKeytab(); + try { + responseBody = connectUgi.doAs(new PrivilegedExceptionAction(){ + @Override + public String run() throws Exception { + AuthenticatedURL.Token token = null; + HttpGet httpGet = new HttpGet(uriBuilder.build()); + if (isKerberosSupportEnabled && UserGroupInformation.isSecurityEnabled() && ( + delegationToken == null || delegationToken.isEmpty())) { + token = new AuthenticatedURL.Token(); + final Authenticator kerberosAuthenticator = new KerberosDelegationTokenAuthenticator(); + kerberosAuthenticator + .authenticate(uriBuilder.build().toURL(), token); + Validate.isTrue(token.isSet(), + "Authenticated Token is NOT present. The request cannot proceed."); + if(token != null){ + httpGet.setHeader("Cookie", AuthenticatedURL.AUTH_COOKIE + "=" + token); + } + } + return remoteCallHelper.makeRemoteGetRequest(httpGet); + }}); + } catch (InterruptedException e) { + LOG.error("Error in check authorization", e); + } + + ObjectMapper objectMapper = new ObjectMapper(); + RemoteAuthorizerResponse authorizerResponse = + objectMapper.readValue(responseBody, RemoteAuthorizerResponse.class); + + if (authorizerResponse == null) { + throw new WasbAuthorizationException( + "RemoteAuthorizerResponse object null from remote call"); + } else if (authorizerResponse.getResponseCode() + == REMOTE_CALL_SUCCESS_CODE) { + return authorizerResponse.getAuthorizationResult(); + } else { + throw new WasbAuthorizationException("Remote authorization" + + " serivce encountered an error " + + authorizerResponse.getResponseMessage()); + } + } catch (URISyntaxException | WasbRemoteCallException + | JsonParseException | JsonMappingException ex) { + throw new WasbAuthorizationException(ex); + } + } +} + +/** + * POJO representing the response expected from a remote + * authorization service. + * The remote service is expected to return the authorization + * response in the following JSON format + * { + * "responseCode" : 0 or non-zero , + * "responseMessage" : relavant message of failure + * "authorizationResult" : authorization result + * true - if auhorization allowed + * false - otherwise. + * + * } + */ +class RemoteAuthorizerResponse { + + private int responseCode; + private boolean authorizationResult; + private String responseMessage; + + public RemoteAuthorizerResponse(){ + } + + public RemoteAuthorizerResponse(int responseCode, + boolean authorizationResult, String message) { + this.responseCode = responseCode; + this.authorizationResult = authorizationResult; + this.responseMessage = message; + } + + public int getResponseCode() { + return responseCode; + } + + public void setResponseCode(int responseCode) { + this.responseCode = responseCode; + } + + public boolean getAuthorizationResult() { + return authorizationResult; + } + + public void setAuthorizationResult(boolean authorizationResult) { + this.authorizationResult = authorizationResult; + } + + public String getResponseMessage() { + return responseMessage; + } + + public void setResponseMessage(String message) { + this.responseMessage = message; + } +} \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java index 6749a76f7f4..5ec9136a9a5 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java @@ -69,19 +69,17 @@ public class SecureStorageInterfaceImpl extends StorageInterface { public static final String SAS_ERROR_CODE = "SAS Error"; private SASKeyGeneratorInterface sasKeyGenerator; private String storageAccount; - private String delegationToken; public SecureStorageInterfaceImpl(boolean useLocalSASKeyMode, - Configuration conf, String delegationToken) + Configuration conf) throws SecureModeException { - this.delegationToken = delegationToken; if (useLocalSASKeyMode) { this.sasKeyGenerator = new LocalSASKeyGeneratorImpl(conf); } else { RemoteSASKeyGeneratorImpl remoteSasKeyGenerator = new RemoteSASKeyGeneratorImpl(conf); - if (!remoteSasKeyGenerator.initialize(conf, this.delegationToken)) { + if (!remoteSasKeyGenerator.initialize(conf)) { throw new SecureModeException("Remote SAS Key mode could" + " not be initialized"); } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationException.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationException.java new file mode 100644 index 00000000000..eff9248dffe --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationException.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure; + +/** + * Exception that gets thrown during the authorization failures + * in WASB. + */ +public class WasbAuthorizationException extends AzureException { + + private static final long serialVersionUID = 1L; + + public WasbAuthorizationException(String message) { + super(message); + } + + public WasbAuthorizationException(String message, Throwable cause) { + super(message, cause); + } + + public WasbAuthorizationException(Throwable t) { + super(t); + } +} \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationOperations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationOperations.java new file mode 100644 index 00000000000..bd768372215 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationOperations.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure; + +/** + * Different authorization operations supported + * in WASB. + */ + +public enum WasbAuthorizationOperations { + + READ, WRITE, EXECUTE; + + @Override + public String toString() { + switch(this) { + case READ: + return "read"; + case WRITE: + return "write"; + case EXECUTE: + return "execute"; + default: + throw new IllegalArgumentException( + "Invalid Auhtorization Operation"); + } + } +} \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizerInterface.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizerInterface.java new file mode 100644 index 00000000000..0c61997a61e --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizerInterface.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; + +/** + * Interface to implement authorization support in WASB. + * API's of this interface will be implemented in the + * StorageInterface Layer before making calls to Azure + * Storage. + */ +public interface WasbAuthorizerInterface { + /** + * Initializer method + * @param conf - Configuration object + */ + public void init(Configuration conf) + throws WasbAuthorizationException, IOException; + + /** + * Authorizer API to authorize access in WASB. + * @param wasbAbsolutePath : Absolute WASB Path used for access. + * @param accessType : Type of access + * @return : true - If access allowed false - If access is not allowed. + */ + public boolean authorize(String wasbAbsolutePath, String accessType) + throws WasbAuthorizationException, IOException; +} \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/Constants.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/Constants.java new file mode 100644 index 00000000000..105068710d5 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/Constants.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure.security; + +/** + * Constants for used with WASB security implementation. + */ +public final class Constants { + + private Constants() { + } + + /** + * Configuration parameter name expected in the Configuration + * object to provide the url of the remote service {@value} + */ + public static final String KEY_CRED_SERVICE_URL = "fs.azure.cred.service.url"; + /** + * Default port of the remote service used as delegation token manager and Azure storage SAS key generator. + */ + public static final int DEFAULT_CRED_SERVICE_PORT = 50911; + + /** + * Default remote delegation token manager endpoint. + */ + public static final String DEFAULT_DELEGATION_TOKEN_MANAGER_ENDPOINT = "/tokenmanager/v1"; + + /** + * The configuration property to enable Kerberos support. + */ + + public static final String AZURE_KERBEROS_SUPPORT_PROPERTY_NAME = "fs.azure.enable.kerberos.support"; + + /** + * Parameter to be used for impersonation. + */ + public static final String DOAS_PARAM="doas"; +} diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbDelegationTokenIdentifier.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbDelegationTokenIdentifier.java new file mode 100644 index 00000000000..530e04572e2 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbDelegationTokenIdentifier.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure.security; + +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier; + +/** + * Delegation token Identifier for WASB delegation tokens. + */ +public class WasbDelegationTokenIdentifier extends DelegationTokenIdentifier { + public static final Text TOKEN_KIND = new Text("WASB delegation"); + + public WasbDelegationTokenIdentifier(){ + super(TOKEN_KIND); + } + + public WasbDelegationTokenIdentifier(Text kind) { + super(kind); + } + + public WasbDelegationTokenIdentifier(Text kind, Text owner, Text renewer, + Text realUser) { + super(kind, owner, renewer, realUser); + } + + @Override + public Text getKind() { + return TOKEN_KIND; + } + +} diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbTokenRenewer.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbTokenRenewer.java new file mode 100644 index 00000000000..09b7349e120 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbTokenRenewer.java @@ -0,0 +1,124 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure.security; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenRenewer; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator; +import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.URL; +import java.security.PrivilegedExceptionAction; + +/** + * Token Renewer for renewing WASB delegation tokens with remote service. + */ +public class WasbTokenRenewer extends TokenRenewer { + public static final Logger LOG = LoggerFactory + .getLogger(WasbTokenRenewer.class); + + @Override + public boolean handleKind(Text kind) { + return WasbDelegationTokenIdentifier.TOKEN_KIND.equals(kind); + } + + @Override + public boolean isManaged(Token token) throws IOException { + return true; + } + + @Override + public long renew(final Token token, Configuration conf) + throws IOException, InterruptedException { + LOG.debug("Renewing the delegation token"); + final UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + UserGroupInformation connectUgi = ugi.getRealUser(); + final UserGroupInformation proxyUser = connectUgi; + if (connectUgi == null) { + connectUgi = ugi; + } + if(!connectUgi.hasKerberosCredentials()){ + connectUgi = UserGroupInformation.getLoginUser(); + } + connectUgi.checkTGTAndReloginFromKeytab(); + final DelegationTokenAuthenticatedURL.Token authToken = new DelegationTokenAuthenticatedURL.Token(); + authToken + .setDelegationToken((Token) token); + final String credServiceUrl = conf.get(Constants.KEY_CRED_SERVICE_URL, + String.format("http://%s:%s", + InetAddress.getLocalHost().getCanonicalHostName(), + Constants.DEFAULT_CRED_SERVICE_PORT)); + DelegationTokenAuthenticator authenticator = new KerberosDelegationTokenAuthenticator(); + final DelegationTokenAuthenticatedURL authURL = new DelegationTokenAuthenticatedURL( + authenticator); + + return connectUgi.doAs(new PrivilegedExceptionAction() { + @Override + public Long run() throws Exception { + return authURL.renewDelegationToken(new URL(credServiceUrl + + Constants.DEFAULT_DELEGATION_TOKEN_MANAGER_ENDPOINT), + authToken, (proxyUser != null) ? ugi.getShortUserName() : null); + } + }); + } + + @Override + public void cancel(final Token token, Configuration conf) + throws IOException, InterruptedException { + LOG.debug("Cancelling the delegation token"); + final UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + UserGroupInformation connectUgi = ugi.getRealUser(); + final UserGroupInformation proxyUser = connectUgi; + if (connectUgi == null) { + connectUgi = ugi; + } + if(!connectUgi.hasKerberosCredentials()){ + connectUgi = UserGroupInformation.getLoginUser(); + } + connectUgi.checkTGTAndReloginFromKeytab(); + final DelegationTokenAuthenticatedURL.Token authToken = new DelegationTokenAuthenticatedURL.Token(); + authToken + .setDelegationToken((Token) token); + final String credServiceUrl = conf.get(Constants.KEY_CRED_SERVICE_URL, + String.format("http://%s:%s", + InetAddress.getLocalHost().getCanonicalHostName(), + Constants.DEFAULT_CRED_SERVICE_PORT)); + DelegationTokenAuthenticator authenticator = new KerberosDelegationTokenAuthenticator(); + final DelegationTokenAuthenticatedURL authURL = new DelegationTokenAuthenticatedURL( + authenticator); + connectUgi.doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + authURL.cancelDelegationToken(new URL(credServiceUrl + + Constants.DEFAULT_DELEGATION_TOKEN_MANAGER_ENDPOINT), + authToken, (proxyUser != null) ? ugi.getShortUserName() : null); + return null; + } + }); + } +} \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/package.html b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/package.html new file mode 100644 index 00000000000..fe58c0a272a --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/package.html @@ -0,0 +1,28 @@ + + + + + + +

+ Infrastructure for WASB client Security to work with Kerberos and Delegation + tokens. +

+ + + diff --git a/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier b/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier new file mode 100644 index 00000000000..7ec8216deb0 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.hadoop.fs.azure.security.WasbDelegationTokenIdentifier \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer b/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer new file mode 100644 index 00000000000..f9c590aad8d --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.hadoop.fs.azure.security.WasbTokenRenewer \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/index.md b/hadoop-tools/hadoop-azure/src/site/markdown/index.md index 2865223acdb..a06d297ea7c 100644 --- a/hadoop-tools/hadoop-azure/src/site/markdown/index.md +++ b/hadoop-tools/hadoop-azure/src/site/markdown/index.md @@ -333,6 +333,40 @@ The service is expected to return a response in JSON format: "sasKey" : Requested SAS Key } ``` + +## Authorization Support in WASB. + +Authorization support can be enabled in WASB using the following configuration: + +``` + + fs.azure.authorization + true + +``` + The current implementation of authorization relies on the presence of an external service that can enforce + the authorization. The service is expected to be running on a URL provided by the following config. + +``` + + fs.azure.authorization.remote.service.url + {URL} + +``` + + The remote service is expected to provide support for the following REST call: ```{URL}/CHECK_AUTHORIZATION``` + An example request: + ```{URL}/CHECK_AUTHORIZATION?wasb_absolute_path=&operation_type=&delegation_token=``` + + The service is expected to return a response in JSON format: + ``` + { + "responseCode" : 0 or non-zero , + "responseMessage" : relavant message on failure , + "authorizationResult" : true/false + } + ``` + ## Testing the hadoop-azure Module The hadoop-azure module includes a full suite of unit tests. Most of the tests diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java new file mode 100644 index 00000000000..af5a537ce4d --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; + +/** + * A mock wasb authorizer implementation. + */ + +public class MockWasbAuthorizerImpl implements WasbAuthorizerInterface { + + private Map authRules; + + @Override + public void init(Configuration conf) { + authRules = new HashMap(); + } + + public void addAuthRule(String wasbAbsolutePath, + String accessType, boolean access) { + AuthorizationComponent component = + new AuthorizationComponent(wasbAbsolutePath, accessType); + this.authRules.put(component, access); + } + + @Override + public boolean authorize(String wasbAbsolutePath, String accessType) + throws WasbAuthorizationException { + + AuthorizationComponent component = + new AuthorizationComponent(wasbAbsolutePath, accessType); + + if (authRules.containsKey(component)) { + return authRules.get(component); + } else { + return false; + } + } +} + +class AuthorizationComponent { + + private String wasbAbsolutePath; + private String accessType; + + public AuthorizationComponent(String wasbAbsolutePath, + String accessType) { + this.wasbAbsolutePath = wasbAbsolutePath; + this.accessType = accessType; + } + + @Override + public int hashCode() { + return this.wasbAbsolutePath.hashCode() ^ this.accessType.hashCode(); + } + + @Override + public boolean equals(Object obj) { + + if (obj == this) { + return true; + } + + if (obj == null + || !(obj instanceof AuthorizationComponent)) { + return false; + } + + return ((AuthorizationComponent)obj). + getWasbAbsolutePath().equals(this.wasbAbsolutePath) + && ((AuthorizationComponent)obj). + getAccessType().equals(this.accessType); + } + + public String getWasbAbsolutePath() { + return this.wasbAbsolutePath; + } + + public String getAccessType() { + return accessType; + } +} \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java new file mode 100644 index 00000000000..e76533550b4 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java @@ -0,0 +1,277 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.Path; +import org.junit.Test; + +import com.sun.tools.javac.util.Assert; + +/** + * Test class to hold all WASB authorization tests. + */ +public class TestNativeAzureFileSystemAuthorization + extends AbstractWasbTestBase { + + @Override + protected AzureBlobStorageTestAccount createTestAccount() throws Exception { + Configuration conf = new Configuration(); + conf.set(NativeAzureFileSystem.KEY_AZURE_AUTHORIZATION, "true"); + conf.set(RemoteWasbAuthorizerImpl.KEY_REMOTE_AUTH_SERVICE_URL, "test_url"); + return AzureBlobStorageTestAccount.create(conf); + } + + /** + * Positive test to verify Create and delete access check + * @throws Throwable + */ + @Test + public void testCreateAccessCheckPositive() throws Throwable { + + AzureBlobStorageTestAccount testAccount = createTestAccount(); + NativeAzureFileSystem fs = testAccount.getFileSystem(); + + String testFile = "test.dat"; + Path testPath = new Path(fs.getWorkingDirectory(), testFile); + + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + authorizer.init(null); + authorizer.addAuthRule(testPath.toString(), + WasbAuthorizationOperations.WRITE.toString(), true); + authorizer.addAuthRule(testPath.toString(), + WasbAuthorizationOperations.EXECUTE.toString(), true); + fs.updateWasbAuthorizer(authorizer); + authorizer.addAuthRule(fs.getWorkingDirectory().toString(), + WasbAuthorizationOperations.EXECUTE.toString(), true); + + fs.create(testPath); + Assert.check(fs.exists(testPath)); + fs.delete(testPath, false); + } + + /** + * Negative test to verify Create access check + * @throws Throwable + */ + + @Test(expected=WasbAuthorizationException.class) + public void testCreateAccessCheckNegative() throws Throwable { + + AzureBlobStorageTestAccount testAccount = createTestAccount(); + NativeAzureFileSystem fs = testAccount.getFileSystem(); + + String testFile = "test.dat"; + Path testPath = new Path(fs.getWorkingDirectory(), testFile); + + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + authorizer.init(null); + authorizer.addAuthRule(testPath.toString(), + WasbAuthorizationOperations.WRITE.toString(), false); + fs.updateWasbAuthorizer(authorizer); + + fs.create(new Path(testFile)); + } + + /** + * Positive test to verify Create and delete access check + * @throws Throwable + */ + @Test + public void testListAccessCheckPositive() throws Throwable { + + AzureBlobStorageTestAccount testAccount = createTestAccount(); + NativeAzureFileSystem fs = testAccount.getFileSystem(); + + String testFolder = "\\"; + Path testPath = new Path(fs.getWorkingDirectory(), testFolder); + + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + authorizer.init(null); + authorizer.addAuthRule(testPath.toString(), + WasbAuthorizationOperations.EXECUTE.toString(), true); + fs.updateWasbAuthorizer(authorizer); + + fs.listStatus(testPath); + } + + /** + * Negative test to verify Create access check + * @throws Throwable + */ + + @Test(expected=WasbAuthorizationException.class) + public void testListAccessCheckNegative() throws Throwable { + + AzureBlobStorageTestAccount testAccount = createTestAccount(); + NativeAzureFileSystem fs = testAccount.getFileSystem(); + + String testFolder = "\\"; + Path testPath = new Path(fs.getWorkingDirectory(), testFolder); + + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + authorizer.init(null); + authorizer.addAuthRule(testPath.toString(), + WasbAuthorizationOperations.EXECUTE.toString(), false); + fs.updateWasbAuthorizer(authorizer); + + fs.listStatus(testPath); + } + + /** + * Positive test to verify rename access check. + * @throws Throwable + */ + @Test + public void testRenameAccessCheckPositive() throws Throwable { + + AzureBlobStorageTestAccount testAccount = createTestAccount(); + NativeAzureFileSystem fs = testAccount.getFileSystem(); + + String testFile = "test.dat"; + Path testPath = new Path(fs.getWorkingDirectory(), testFile); + String renameFile = "test2.dat"; + Path renamePath = new Path(fs.getWorkingDirectory(), renameFile); + + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + authorizer.init(null); + authorizer.addAuthRule(testPath.toString(), + WasbAuthorizationOperations.WRITE.toString(), true); + authorizer.addAuthRule(testPath.toString(), + WasbAuthorizationOperations.EXECUTE.toString(), true); + authorizer.addAuthRule(renamePath.toString(), + WasbAuthorizationOperations.EXECUTE.toString(), true); + authorizer.addAuthRule(fs.getWorkingDirectory().toString(), + WasbAuthorizationOperations.EXECUTE.toString(), true); + fs.updateWasbAuthorizer(authorizer); + fs.create(testPath); + + Assert.check(fs.exists(testPath)); + fs.rename(testPath, renamePath); + Assert.check(fs.exists(renamePath)); + fs.delete(renamePath, false); + } + + /** + * Negative test to verify rename access check. + * @throws Throwable + */ + @Test(expected=WasbAuthorizationException.class) + public void testRenameAccessCheckNegative() throws Throwable { + + AzureBlobStorageTestAccount testAccount = createTestAccount(); + NativeAzureFileSystem fs = testAccount.getFileSystem(); + String testFile = "test.dat"; + Path testPath = new Path(fs.getWorkingDirectory(), testFile); + Path renamePath = new Path("test2.dat"); + + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + authorizer.init(null); + authorizer.addAuthRule(testPath.toString(), + WasbAuthorizationOperations.WRITE.toString(), true); + authorizer.addAuthRule(testPath.toString(), + WasbAuthorizationOperations.EXECUTE.toString(), false); + fs.updateWasbAuthorizer(authorizer); + + try { + fs.create(testPath); + + Assert.check(fs.exists(testPath)); + fs.rename(testPath, renamePath); + Assert.check(fs.exists(renamePath)); + fs.delete(renamePath, false); + } catch (WasbAuthorizationException ex) { + throw ex; + } finally { + authorizer = new MockWasbAuthorizerImpl(); + authorizer.init(null); + authorizer.addAuthRule(testPath.toString(), + WasbAuthorizationOperations.EXECUTE.toString(), false); + fs.updateWasbAuthorizer(authorizer); + Assert.check(fs.exists(testPath)); + fs.delete(testPath, false); + } + } + + /** + * Positive test for read access check. + * @throws Throwable + */ + @Test + public void testReadAccessCheckPositive() throws Throwable { + + AzureBlobStorageTestAccount testAccount = createTestAccount(); + NativeAzureFileSystem fs = testAccount.getFileSystem(); + String testFile = "test.dat"; + Path testPath = new Path(fs.getWorkingDirectory(), testFile); + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + authorizer.init(null); + authorizer.addAuthRule(testPath.toString(), + WasbAuthorizationOperations.WRITE.toString(), true); + authorizer.addAuthRule(testPath.toString(), + WasbAuthorizationOperations.EXECUTE.toString(), true); + authorizer.addAuthRule(testPath.toString(), + WasbAuthorizationOperations.READ.toString(), true); + authorizer.addAuthRule(fs.getWorkingDirectory().toString(), + WasbAuthorizationOperations.EXECUTE.toString(), true); + fs.updateWasbAuthorizer(authorizer); + fs.create(testPath); + Assert.check(fs.exists(testPath)); + FSDataInputStream inputStream = fs.open(testPath); + inputStream.close(); + fs.delete(testPath, false); + } + + /** + * Negative test to verify read access check. + * @throws Throwable + */ + @Test(expected=WasbAuthorizationException.class) + public void testReadAccessCheckNegative() throws Throwable { + + AzureBlobStorageTestAccount testAccount = createTestAccount(); + NativeAzureFileSystem fs = testAccount.getFileSystem(); + String testFile = "test.dat"; + Path testPath = new Path(fs.getWorkingDirectory(), testFile); + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + authorizer.init(null); + authorizer.addAuthRule(testPath.toString(), + WasbAuthorizationOperations.WRITE.toString(), true); + authorizer.addAuthRule(testPath.toString(), + WasbAuthorizationOperations.EXECUTE.toString(), true); + authorizer.addAuthRule(testPath.toString(), + WasbAuthorizationOperations.READ.toString(), false); + fs.updateWasbAuthorizer(authorizer); + + fs.create(new Path(testFile)); + Assert.check(fs.exists(testPath)); + FSDataInputStream inputStream = null; + try { + inputStream = fs.open(new Path(testFile)); + } catch (WasbAuthorizationException ex) { + throw ex; + } finally { + fs.delete(new Path(testFile), false); + if (inputStream != null) { + inputStream.close(); + } + } + } +} \ No newline at end of file From 0f336bab9c070e0cdc935df4305084a036e2def7 Mon Sep 17 00:00:00 2001 From: Mingliang Liu Date: Fri, 3 Mar 2017 18:47:13 -0800 Subject: [PATCH 007/188] HDFS-11494. Log message when DN is not selected for block replication. Contributed by Yiqun Lin --- .../hdfs/server/blockmanagement/DatanodeDescriptor.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java index 15af6ab1a7f..7b3e4e17d5d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java @@ -687,6 +687,10 @@ public class DatanodeDescriptor extends DatanodeInfo { } } if (requiredSize > remaining - scheduledSize) { + LOG.debug( + "The node {} does not have enough {} space (required={}," + + " scheduled={}, remaining={}).", + this, t, requiredSize, scheduledSize, remaining); return null; } return storage; From c61bc2d843ace614078be24b2dcbf3fb43d6f358 Mon Sep 17 00:00:00 2001 From: Wangda Tan Date: Fri, 3 Mar 2017 23:43:25 -0800 Subject: [PATCH 008/188] YARN-6278. Enforce to use correct node and npm version in new YARN-UI build. (Sunil G via wangda) --- .../hadoop-yarn/hadoop-yarn-ui/pom.xml | 68 ++++++++++--------- .../hadoop-yarn-ui/src/main/webapp/bower.json | 2 + .../src/main/webapp/package.json | 11 +-- 3 files changed, 45 insertions(+), 36 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml index 22031fa43b7..ae7ab0d3c71 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml @@ -32,8 +32,8 @@ pom ${basedir}/target/src/main/webapp node - v0.12.2 - 2.10.0 + v4.4.5 + 2.15.5 false @@ -120,41 +120,47 @@ + + com.github.eirslett + frontend-maven-plugin + + ${webappTgtDir} + + + + generate-sources + install node and npm + + install-node-and-npm + + + ${nodeVersion} + ${npmVersion} + + + + generate-sources + npm install + + npm + + + + generate-sources + bower install + + bower + + + + + exec-maven-plugin org.codehaus.mojo - - generate-sources - npm install - - exec - - - ${webappTgtDir} - npm - - install - - - - - generate-sources - bower install - - exec - - - ${webappTgtDir} - bower - - --allow-root - install - - - ember build generate-sources diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/bower.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/bower.json index fe5f289ad8f..e1ab943fc18 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/bower.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/bower.json @@ -12,6 +12,8 @@ "loader.js": "3.3.0", "qunit": "1.19.0", "jquery-ui": "1.11.4", + "moment": "2.12.0", + "moment-timezone": "0.5.0", "more-js": "0.8.2", "bootstrap": "3.3.6", "d3": "~3.5.6", diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package.json index 4cd0fdc18ad..573e5051283 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package.json @@ -18,12 +18,13 @@ "author": "", "license": "Apache", "devDependencies": { + "bower": "1.7.7", "broccoli-asset-rev": "2.4.2", "broccoli-funnel": "1.0.1", - "em-table": "0.7.0", + "broccoli-merge-trees": "1.1.1", "ember-array-contains-helper": "1.0.2", "ember-bootstrap": "0.5.1", - "ember-cli": "1.13.13", + "ember-cli": "^1.13.13", "ember-cli-app-version": "1.0.0", "ember-cli-babel": "5.1.6", "ember-cli-content-security-policy": "0.4.0", @@ -33,6 +34,8 @@ "ember-cli-ic-ajax": "0.2.1", "ember-cli-inject-live-reload": "1.4.0", "ember-cli-jquery-ui": "0.0.20", + "ember-cli-moment-shim": "0.7.3", + "ember-cli-numeral": "^0.2.0", "ember-cli-qunit": "1.2.1", "ember-cli-release": "0.2.8", "ember-cli-sri": "1.2.1", @@ -48,8 +51,6 @@ }, "dependencies": { "em-helpers": "^0.8.0", - "em-table": "^0.7.0", - "ember-cli-moment-shim": "^3.0.1", - "ember-cli-numeral": "^0.2.0" + "em-table": "^0.7.0" } } From c1386a90d7ff65b57be00a7245de82c31abb4665 Mon Sep 17 00:00:00 2001 From: Sunil G Date: Sat, 4 Mar 2017 13:46:35 +0530 Subject: [PATCH 009/188] =?UTF-8?q?YARN-6248.=20user=20is=20not=20removed?= =?UTF-8?q?=20from=20UsersManager=E2=80=99s=20when=20app=20is=20killed=20w?= =?UTF-8?q?ith=20pending=20container=20requests.=20Contributed=20by=20Eric?= =?UTF-8?q?=20Payne.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../resourcemanager/scheduler/capacity/UsersManager.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java index 05503c6263d..c2134eb3309 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java @@ -872,7 +872,9 @@ public class UsersManager implements AbstractUsersManager { this.writeLock.lock(); // For UT case: We might need to add the user to users list. - User user = getUserAndAddIfAbsent(userName); + User user = getUser(userName); + if (user == null) return; + ResourceUsage resourceUsage = user.getResourceUsage(); // If User is moved to non-active list, moved resource usage from // non-active to active list. From c8bd8ac7f9642b8db5949a914f209d5de3d4ef9b Mon Sep 17 00:00:00 2001 From: Daniel Templeton Date: Sat, 4 Mar 2017 11:00:03 -0800 Subject: [PATCH 010/188] =?UTF-8?q?HADOOP-14026.=20start-build-env.sh:=20i?= =?UTF-8?q?nvalid=20docker=20image=20name=20(Contributed=20by=20Gerg=C5=91?= =?UTF-8?q?=20P=C3=A1sztor=20via=20Daniel=20Templeton)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- start-build-env.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/start-build-env.sh b/start-build-env.sh index 1fa084f7796..18e3a8c38c4 100755 --- a/start-build-env.sh +++ b/start-build-env.sh @@ -31,7 +31,7 @@ else # boot2docker uid and gid GROUP_ID=50 fi -docker build -t "hadoop-build-${USER_NAME}" - < Date: Mon, 6 Mar 2017 10:30:01 +0900 Subject: [PATCH 011/188] HADOOP-14056. Update maven-javadoc-plugin to 2.10.4. --- hadoop-project/pom.xml | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index c8aa85777de..4c8b9006471 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -1322,7 +1322,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 2.8.1 + ${maven-javadoc-plugin.version} -Xmaxwarns 10000 diff --git a/pom.xml b/pom.xml index 3eeba1e793e..f07256da89d 100644 --- a/pom.xml +++ b/pom.xml @@ -98,7 +98,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs 2.4 2.10 1.4.1 - 2.9.1 + 2.10.4 1.5 1.5 3.0.1 From 3536ce031ca780d6de83cf67779f571a0142ccc8 Mon Sep 17 00:00:00 2001 From: Yiqun Lin Date: Mon, 6 Mar 2017 19:04:03 +0800 Subject: [PATCH 012/188] HDFS-8741. Proper error msg to be printed when invalid operation type is given to WebHDFS operations. Contributed by Surendra Singh Lilhore. --- .../hdfs/web/resources/DeleteOpParam.java | 11 ++++- .../hadoop/hdfs/web/resources/GetOpParam.java | 11 ++++- .../hdfs/web/resources/PostOpParam.java | 11 ++++- .../hadoop/hdfs/web/resources/PutOpParam.java | 11 ++++- .../hadoop/hdfs/web/resources/TestParam.java | 41 +++++++++++++++++++ 5 files changed, 81 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/DeleteOpParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/DeleteOpParam.java index 25bed1c6d3e..e765498b5f0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/DeleteOpParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/DeleteOpParam.java @@ -72,7 +72,16 @@ public class DeleteOpParam extends HttpOpParam { * @param str a string representation of the parameter value. */ public DeleteOpParam(final String str) { - super(DOMAIN, DOMAIN.parse(str)); + super(DOMAIN, getOp(str)); + } + + private static Op getOp(String str) { + try { + return DOMAIN.parse(str); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException(str + " is not a valid " + Type.DELETE + + " operation."); + } } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java index 1321bf6fad9..d32af330ae7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java @@ -111,7 +111,16 @@ public class GetOpParam extends HttpOpParam { * @param str a string representation of the parameter value. */ public GetOpParam(final String str) { - super(DOMAIN, DOMAIN.parse(str)); + super(DOMAIN, getOp(str)); + } + + private static Op getOp(String str) { + try { + return DOMAIN.parse(str); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException(str + " is not a valid " + Type.GET + + " operation."); + } } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PostOpParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PostOpParam.java index 56a14c7cc4a..305db46e9c2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PostOpParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PostOpParam.java @@ -80,7 +80,16 @@ public class PostOpParam extends HttpOpParam { * @param str a string representation of the parameter value. */ public PostOpParam(final String str) { - super(DOMAIN, DOMAIN.parse(str)); + super(DOMAIN, getOp(str)); + } + + private static Op getOp(String str) { + try { + return DOMAIN.parse(str); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException(str + " is not a valid " + Type.POST + + " operation."); + } } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PutOpParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PutOpParam.java index 4bb48a62288..558bb53a429 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PutOpParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PutOpParam.java @@ -107,7 +107,16 @@ public class PutOpParam extends HttpOpParam { * @param str a string representation of the parameter value. */ public PutOpParam(final String str) { - super(DOMAIN, DOMAIN.parse(str)); + super(DOMAIN, getOp(str)); + } + + private static Op getOp(String str) { + try { + return DOMAIN.parse(str); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException(str + " is not a valid " + Type.PUT + + " operation."); + } } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java index d444cb4360a..6c145a4b6c0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java @@ -36,6 +36,7 @@ import org.apache.hadoop.fs.XAttrSetFlag; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.StringUtils; import org.junit.Assert; import org.junit.Test; @@ -503,4 +504,44 @@ public class TestParam { p = new StoragePolicyParam("COLD"); Assert.assertEquals("COLD", p.getValue()); } + + @Test + public void testHttpOpParams() { + try { + new PostOpParam("TEST"); + Assert + .fail("Construct the PostOpParam with param value 'TEST' should be" + + " failed."); + } catch (IllegalArgumentException e) { + GenericTestUtils.assertExceptionContains( + "TEST is not a valid POST operation.", e); + } + try { + new PutOpParam("TEST"); + Assert + .fail("Construct the PutOpParam with param value 'TEST' should be" + + " failed."); + } catch (IllegalArgumentException e) { + GenericTestUtils.assertExceptionContains( + "TEST is not a valid PUT operation.", e); + } + try { + new DeleteOpParam("TEST"); + Assert + .fail("Construct the DeleteOpParam with param value 'TEST' should be" + + " failed."); + } catch (IllegalArgumentException e) { + GenericTestUtils.assertExceptionContains( + "TEST is not a valid DELETE operation.", e); + } + try { + new GetOpParam("TEST"); + Assert + .fail("Construct the GetOpParam with param value 'TEST' should be" + + " failed."); + } catch (IllegalArgumentException e) { + GenericTestUtils.assertExceptionContains( + "TEST is not a valid GET operation.", e); + } + } } From 209ecd1a5c056e096a5d804759f88302608d503b Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Tue, 7 Mar 2017 01:32:47 +0900 Subject: [PATCH 013/188] HADOOP-14108. CLI MiniCluster: add an option to specify NameNode HTTP port. Contributed by Takanobu Asanuma. --- .../hadoop-common/src/site/markdown/CLIMiniCluster.md.vm | 1 + .../apache/hadoop/mapreduce/MiniHadoopClusterManager.java | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/CLIMiniCluster.md.vm b/hadoop-common-project/hadoop-common/src/site/markdown/CLIMiniCluster.md.vm index 74c241400fa..806df0a41dc 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/CLIMiniCluster.md.vm +++ b/hadoop-common-project/hadoop-common/src/site/markdown/CLIMiniCluster.md.vm @@ -55,6 +55,7 @@ There are a number of command line arguments that the users can use to control w $ -namenode URL of the namenode (default is either the DFS $ cluster or a temporary dir) $ -nnport NameNode port (default 0--we choose) + $ -nnhttpport NameNode HTTP port (default 0--we choose) $ -nodemanagers How many nodemanagers to start (default 1) $ -nodfs Don't start a mini DFS cluster $ -nomr Don't start a mini MR cluster diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java index 3cc73b5abad..324f0ca0322 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java @@ -69,6 +69,7 @@ public class MiniHadoopClusterManager { private int numNodeManagers; private int numDataNodes; private int nnPort; + private int nnHttpPort; private int rmPort; private int jhsPort; private StartupOption dfsOpts; @@ -92,6 +93,8 @@ public class MiniHadoopClusterManager { .addOption("datanodes", true, "How many datanodes to start (default 1)") .addOption("format", false, "Format the DFS (default false)") .addOption("nnport", true, "NameNode port (default 0--we choose)") + .addOption("nnhttpport", true, + "NameNode HTTP port (default 0--we choose)") .addOption( "namenode", true, @@ -152,7 +155,8 @@ public class MiniHadoopClusterManager { URISyntaxException { if (!noDFS) { dfs = new MiniDFSCluster.Builder(conf).nameNodePort(nnPort) - .numDataNodes(numDataNodes).startupOption(dfsOpts).build(); + .nameNodeHttpPort(nnHttpPort).numDataNodes(numDataNodes) + .startupOption(dfsOpts).build(); LOG.info("Started MiniDFSCluster -- namenode on port " + dfs.getNameNodePort()); } @@ -254,6 +258,7 @@ public class MiniHadoopClusterManager { noDFS = cli.hasOption("nodfs"); numDataNodes = intArgument(cli, "datanodes", 1); nnPort = intArgument(cli, "nnport", 0); + nnHttpPort = intArgument(cli, "nnhttpport", 0); dfsOpts = cli.hasOption("format") ? StartupOption.FORMAT : StartupOption.REGULAR; From ec839b94c0eb3f09e74f8a3b0bc9a08b3f5418b2 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Mon, 6 Mar 2017 10:47:15 -0800 Subject: [PATCH 014/188] HDFS-11441. Add escaping to error message in KMS web UI. Contributed by Aaron T. Myers. --- .../hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java index 45e48e9a7d8..3e98a252320 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java +++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java @@ -20,6 +20,7 @@ package org.apache.hadoop.crypto.key.kms.server; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.crypto.key.kms.KMSDelegationToken; +import org.apache.hadoop.http.HtmlQuoting; import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler; import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationFilter; @@ -105,7 +106,7 @@ public class KMSAuthenticationFilter public void sendError(int sc, String msg) throws IOException { statusCode = sc; this.msg = msg; - super.sendError(sc, msg); + super.sendError(sc, HtmlQuoting.quoteHtmlChars(msg)); } @Override From d9dc444dc73fbe23f9e553d63baf83f12c636fa7 Mon Sep 17 00:00:00 2001 From: Ray Chiang Date: Mon, 6 Mar 2017 14:02:49 -0800 Subject: [PATCH 015/188] YARN-5665. Enhance documentation for yarn.resourcemanager.scheduler.class property. (Yufei Gu via rchiang) --- .../hadoop-common/src/site/markdown/ClusterSetup.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md b/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md index 1d9e9da5783..7be6a192c9c 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md @@ -139,7 +139,7 @@ This section deals with important parameters to be specified in the given config | `yarn.resourcemanager.admin.address` | `ResourceManager` host:port for administrative commands. | *host:port* If set, overrides the hostname set in `yarn.resourcemanager.hostname`. | | `yarn.resourcemanager.webapp.address` | `ResourceManager` web-ui host:port. | *host:port* If set, overrides the hostname set in `yarn.resourcemanager.hostname`. | | `yarn.resourcemanager.hostname` | `ResourceManager` host. | *host* Single hostname that can be set in place of setting all `yarn.resourcemanager*address` resources. Results in default ports for ResourceManager components. | -| `yarn.resourcemanager.scheduler.class` | `ResourceManager` Scheduler class. | `CapacityScheduler` (recommended), `FairScheduler` (also recommended), or `FifoScheduler` | +| `yarn.resourcemanager.scheduler.class` | `ResourceManager` Scheduler class. | `CapacityScheduler` (recommended), `FairScheduler` (also recommended), or `FifoScheduler`. Use a fully qualified class name, e.g., `org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler`. | | `yarn.scheduler.minimum-allocation-mb` | Minimum limit of memory to allocate to each container request at the `Resource Manager`. | In MBs | | `yarn.scheduler.maximum-allocation-mb` | Maximum limit of memory to allocate to each container request at the `Resource Manager`. | In MBs | | `yarn.resourcemanager.nodes.include-path` / `yarn.resourcemanager.nodes.exclude-path` | List of permitted/excluded NodeManagers. | If necessary, use these files to control the list of allowable NodeManagers. | From 5e74196ede9bfc20eb6d6fe3aa6a0e5c47a40fdd Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Mon, 6 Mar 2017 15:04:13 -0800 Subject: [PATCH 016/188] HDFS-11498. Make RestCsrfPreventionHandler and WebHdfsHandler compatible with Netty 4.0. --- .../web/RestCsrfPreventionFilterHandler.java | 6 ++-- .../datanode/web/webhdfs/WebHdfsHandler.java | 33 +++++++++---------- hadoop-project/pom.xml | 2 +- 3 files changed, 20 insertions(+), 21 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/RestCsrfPreventionFilterHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/RestCsrfPreventionFilterHandler.java index f2f0533894d..4958bb59202 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/RestCsrfPreventionFilterHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/RestCsrfPreventionFilterHandler.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hdfs.server.datanode.web; -import static io.netty.handler.codec.http.HttpHeaderNames.CONNECTION; -import static io.netty.handler.codec.http.HttpHeaderValues.CLOSE; +import static io.netty.handler.codec.http.HttpHeaders.Names.CONNECTION; +import static io.netty.handler.codec.http.HttpHeaders.Values.CLOSE; import static io.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR; import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; @@ -119,7 +119,7 @@ final class RestCsrfPreventionFilterHandler @Override public String getMethod() { - return req.method().name(); + return req.getMethod().name(); } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java index d2b2ec22346..c5fc7ea1709 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java @@ -29,17 +29,6 @@ import io.netty.handler.codec.http.HttpMethod; import io.netty.handler.codec.http.HttpRequest; import io.netty.handler.codec.http.QueryStringDecoder; import io.netty.handler.stream.ChunkedStream; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.InetSocketAddress; -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.charset.StandardCharsets; -import java.security.PrivilegedExceptionAction; -import java.util.EnumSet; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -63,17 +52,27 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.LimitInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.security.PrivilegedExceptionAction; +import java.util.EnumSet; + +import static io.netty.handler.codec.http.HttpHeaders.Names.ACCEPT; +import static io.netty.handler.codec.http.HttpHeaders.Names.ACCESS_CONTROL_ALLOW_HEADERS; import static io.netty.handler.codec.http.HttpHeaders.Names.ACCESS_CONTROL_ALLOW_METHODS; import static io.netty.handler.codec.http.HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN; +import static io.netty.handler.codec.http.HttpHeaders.Names.ACCESS_CONTROL_MAX_AGE; import static io.netty.handler.codec.http.HttpHeaders.Names.CONNECTION; import static io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_LENGTH; import static io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_TYPE; import static io.netty.handler.codec.http.HttpHeaders.Names.LOCATION; import static io.netty.handler.codec.http.HttpHeaders.Values.CLOSE; -import static io.netty.handler.codec.http.HttpHeaderNames.ACCEPT; -import static io.netty.handler.codec.http.HttpHeaderNames.ACCESS_CONTROL_ALLOW_HEADERS; -import static io.netty.handler.codec.http.HttpHeaderNames.ACCESS_CONTROL_MAX_AGE; -import static io.netty.handler.codec.http.HttpHeaderValues.KEEP_ALIVE; +import static io.netty.handler.codec.http.HttpHeaders.Values.KEEP_ALIVE; import static io.netty.handler.codec.http.HttpMethod.GET; import static io.netty.handler.codec.http.HttpMethod.OPTIONS; import static io.netty.handler.codec.http.HttpMethod.POST; @@ -145,7 +144,7 @@ public class WebHdfsHandler extends SimpleChannelInboundHandler { LOG.warn("Error retrieving hostname: ", e); host = "unknown"; } - REQLOG.info(host + " " + req.method() + " " + req.uri() + " " + + REQLOG.info(host + " " + req.getMethod() + " " + req.getUri() + " " + getResponseCode()); } return null; @@ -155,7 +154,7 @@ public class WebHdfsHandler extends SimpleChannelInboundHandler { int getResponseCode() { return (resp == null) ? INTERNAL_SERVER_ERROR.code() : - resp.status().code(); + resp.getStatus().code(); } public void handle(ChannelHandlerContext ctx, HttpRequest req) diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 4c8b9006471..572f0e6eb29 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -719,7 +719,7 @@ io.netty netty-all - 4.1.0.Beta5 + 4.0.23.Final From b5adc5c3011f111f86d232cb33ec522547f68a95 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Mon, 6 Mar 2017 16:39:53 -0800 Subject: [PATCH 017/188] HDFS-10838. Last full block report received time for each DN should be easily discoverable. Contributed by Surendra Singh Lilhore. --- .../hadoop/hdfs/protocol/DatanodeInfo.java | 52 ++++++++++++++++++- .../hdfs/protocolPB/PBHelperClient.java | 8 ++- .../hadoop/hdfs/web/JsonUtilClient.java | 2 + .../src/main/proto/hdfs.proto | 2 + .../server/blockmanagement/BlockManager.java | 3 ++ .../hdfs/server/namenode/FSNamesystem.java | 8 ++- .../org/apache/hadoop/hdfs/web/JsonUtil.java | 3 ++ .../src/main/webapps/hdfs/dfshealth.html | 2 + .../src/main/webapps/hdfs/dfshealth.js | 1 + .../server/namenode/TestNameNodeMXBean.java | 1 + 10 files changed, 78 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java index acbcffa6f03..e1698c98c03 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java @@ -85,6 +85,8 @@ public class DatanodeInfo extends DatanodeID implements Node { protected AdminStates adminState; private long maintenanceExpireTimeInMS; + private long lastBlockReportTime; + private long lastBlockReportMonotonic; protected DatanodeInfo(DatanodeInfo from) { super(from); @@ -101,6 +103,8 @@ public class DatanodeInfo extends DatanodeID implements Node { this.location = from.getNetworkLocation(); this.adminState = from.getAdminState(); this.upgradeDomain = from.getUpgradeDomain(); + this.lastBlockReportTime = from.getLastBlockReportTime(); + this.lastBlockReportMonotonic = from.getLastBlockReportMonotonic(); } protected DatanodeInfo(DatanodeID nodeID) { @@ -116,6 +120,8 @@ public class DatanodeInfo extends DatanodeID implements Node { this.lastUpdateMonotonic = 0L; this.xceiverCount = 0; this.adminState = null; + this.lastBlockReportTime = 0L; + this.lastBlockReportMonotonic = 0L; } protected DatanodeInfo(DatanodeID nodeID, String location) { @@ -131,7 +137,8 @@ public class DatanodeInfo extends DatanodeID implements Node { final long blockPoolUsed, final long cacheCapacity, final long cacheUsed, final long lastUpdate, final long lastUpdateMonotonic, final int xceiverCount, final String networkLocation, - final AdminStates adminState, final String upgradeDomain) { + final AdminStates adminState, final String upgradeDomain, + final long lastBlockReportTime, final long lastBlockReportMonotonic) { super(ipAddr, hostName, datanodeUuid, xferPort, infoPort, infoSecurePort, ipcPort); this.capacity = capacity; @@ -147,6 +154,8 @@ public class DatanodeInfo extends DatanodeID implements Node { this.location = networkLocation; this.adminState = adminState; this.upgradeDomain = upgradeDomain; + this.lastBlockReportTime = lastBlockReportTime; + this.lastBlockReportMonotonic = lastBlockReportMonotonic; } /** Network location name. */ @@ -391,6 +400,11 @@ public class DatanodeInfo extends DatanodeID implements Node { .append(percent2String(cacheRemainingPercent)).append("\n"); buffer.append("Xceivers: ").append(getXceiverCount()).append("\n"); buffer.append("Last contact: ").append(new Date(lastUpdate)).append("\n"); + buffer + .append("Last Block Report: ") + .append( + lastBlockReportTime != 0 ? new Date(lastBlockReportTime) : "Never") + .append("\n"); return buffer.toString(); } @@ -503,6 +517,26 @@ public class DatanodeInfo extends DatanodeID implements Node { return this.maintenanceExpireTimeInMS; } + /** Sets the last block report time. */ + public void setLastBlockReportTime(long lastBlockReportTime) { + this.lastBlockReportTime = lastBlockReportTime; + } + + /** Sets the last block report monotonic time. */ + public void setLastBlockReportMonotonic(long lastBlockReportMonotonic) { + this.lastBlockReportMonotonic = lastBlockReportMonotonic; + } + + /** Last block report time. */ + public long getLastBlockReportTime() { + return lastBlockReportTime; + } + + /** Last block report monotonic time. */ + public long getLastBlockReportMonotonic() { + return lastBlockReportMonotonic; + } + /** * Take the node out of maintenance mode. */ @@ -643,6 +677,8 @@ public class DatanodeInfo extends DatanodeID implements Node { private int infoSecurePort; private int ipcPort; private long nonDfsUsed = 0L; + private long lastBlockReportTime = 0L; + private long lastBlockReportMonotonic = 0L; public DatanodeInfoBuilder setFrom(DatanodeInfo from) { this.capacity = from.getCapacity(); @@ -658,6 +694,8 @@ public class DatanodeInfo extends DatanodeID implements Node { this.location = from.getNetworkLocation(); this.adminState = from.getAdminState(); this.upgradeDomain = from.getUpgradeDomain(); + this.lastBlockReportTime = from.getLastBlockReportTime(); + this.lastBlockReportMonotonic = from.getLastBlockReportMonotonic(); setNodeID(from); return this; } @@ -775,12 +813,22 @@ public class DatanodeInfo extends DatanodeID implements Node { return this; } + public DatanodeInfoBuilder setLastBlockReportTime(long time) { + this.lastBlockReportTime = time; + return this; + } + + public DatanodeInfoBuilder setLastBlockReportMonotonic(long time) { + this.lastBlockReportMonotonic = time; + return this; + } + public DatanodeInfo build() { return new DatanodeInfo(ipAddr, hostName, datanodeUuid, xferPort, infoPort, infoSecurePort, ipcPort, capacity, dfsUsed, nonDfsUsed, remaining, blockPoolUsed, cacheCapacity, cacheUsed, lastUpdate, lastUpdateMonotonic, xceiverCount, location, adminState, - upgradeDomain); + upgradeDomain, lastBlockReportTime, lastBlockReportMonotonic); } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java index ad80bc255a6..b4fa9267fc2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java @@ -303,6 +303,8 @@ public class PBHelperClient { .setLastUpdateMonotonic(info.getLastUpdateMonotonic()) .setXceiverCount(info.getXceiverCount()) .setAdminState(convert(info.getAdminState())) + .setLastBlockReportTime(info.getLastBlockReportTime()) + .setLastBlockReportMonotonic(info.getLastBlockReportMonotonic()) .build(); return builder.build(); } @@ -650,7 +652,11 @@ public class PBHelperClient { .setLastUpdateMonotonic(di.getLastUpdateMonotonic()) .setXceiverCount(di.getXceiverCount()) .setAdminState(convert(di.getAdminState())).setUpgradeDomain( - di.hasUpgradeDomain() ? di.getUpgradeDomain() : null); + di.hasUpgradeDomain() ? di.getUpgradeDomain() : null) + .setLastBlockReportTime(di.hasLastBlockReportTime() ? + di.getLastBlockReportTime() : 0) + .setLastBlockReportMonotonic(di.hasLastBlockReportMonotonic() ? + di.getLastBlockReportMonotonic() : 0); if (di.hasNonDfsUsed()) { dinfo.setNonDfsUsed(di.getNonDfsUsed()); } else { diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java index 4204c54c816..cfcb4c9b07b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java @@ -295,6 +295,8 @@ class JsonUtilClient { DatanodeInfo.AdminStates .valueOf(getString(m, "adminState", "NORMAL"))) .setUpgradeDomain(getString(m, "upgradeDomain", "")) + .setLastBlockReportTime(getLong(m, "lastBlockReportTime", 0L)) + .setLastBlockReportMonotonic(getLong(m, "lastBlockReportMonotonic", 0L)) .build(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto index 8a039d482f4..7824fd11c7a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto @@ -102,6 +102,8 @@ message DatanodeInfoProto { optional uint64 cacheUsed = 12 [default = 0]; optional uint64 lastUpdateMonotonic = 13 [default = 0]; optional string upgradeDomain = 14; + optional uint64 lastBlockReportTime = 15 [default = 0]; + optional uint64 lastBlockReportMonotonic = 16 [default = 0]; } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 5ca0fa7e791..9ec28f9c02f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import static org.apache.hadoop.hdfs.protocol.BlockType.CONTIGUOUS; import static org.apache.hadoop.hdfs.protocol.BlockType.STRIPED; import static org.apache.hadoop.util.ExitUtil.terminate; +import static org.apache.hadoop.util.Time.now; import java.io.IOException; import java.io.PrintWriter; @@ -2377,6 +2378,8 @@ public class BlockManager implements BlockStatsMXBean { long leaseId = this.getBlockReportLeaseManager().removeLease(node); BlockManagerFaultInjector.getInstance(). removeBlockReportLease(node, leaseId); + node.setLastBlockReportTime(now()); + node.setLastBlockReportMonotonic(Time.monotonicNow()); } LOG.debug("Processing RPC with index {} out of total {} RPCs in " + "processReport 0x{}", context.getCurRpc(), diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 81c5759b573..03277af446a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -5472,7 +5472,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, .put("blockScheduled", node.getBlocksScheduled()) .put("blockPoolUsed", node.getBlockPoolUsed()) .put("blockPoolUsedPercent", node.getBlockPoolUsedPercent()) - .put("volfails", node.getVolumeFailures()); + .put("volfails", node.getVolumeFailures()) + // Block report time in minutes + .put("lastBlockReport", getLastBlockReport(node)); VolumeFailureSummary volumeFailureSummary = node.getVolumeFailureSummary(); if (volumeFailureSummary != null) { innerinfo @@ -5571,6 +5573,10 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, return (monotonicNow() - alivenode.getLastUpdateMonotonic())/1000; } + private Object getLastBlockReport(DatanodeDescriptor node) { + return (monotonicNow() - node.getLastBlockReportMonotonic()) / 60000; + } + private long getDfsUsed(DatanodeDescriptor alivenode) { return alivenode.getDfsUsed(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java index 0d891139712..827e1a320ff 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java @@ -181,6 +181,9 @@ public class JsonUtil { if (datanodeinfo.getUpgradeDomain() != null) { m.put("upgradeDomain", datanodeinfo.getUpgradeDomain()); } + m.put("lastBlockReportTime", datanodeinfo.getLastBlockReportTime()); + m.put("lastBlockReportMonotonic", + datanodeinfo.getLastBlockReportMonotonic()); return m; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html index b33b9a27e60..c01f36cdb6b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html @@ -309,6 +309,7 @@ Node Http Address Last contact + Last Block Report Capacity Blocks Block pool used @@ -320,6 +321,7 @@ {name} ({xferaddr}) {dnWebAddress} {lastContact}s + {lastBlockReport}m
{capacity|fmt_bytes}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js index 88096d33f50..59958a1ed1e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js @@ -333,6 +333,7 @@ { 'orderDataType': 'ng-value', 'searchable': true }, { 'orderDataType': 'ng-value', 'type': 'numeric' }, { 'orderDataType': 'ng-value', 'type': 'numeric' }, + { 'orderDataType': 'ng-value', 'type': 'numeric' }, { 'orderData': 3, 'type': 'numeric' }, { 'orderDataType': 'ng-value', 'type': 'numeric'}, { 'orderData': 5 } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java index 037fd4025bd..d4d47637310 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java @@ -174,6 +174,7 @@ public class TestNameNodeMXBean { assertTrue(((Long)liveNode.get("capacity")) > 0); assertTrue(liveNode.containsKey("numBlocks")); assertTrue(((Long)liveNode.get("numBlocks")) == 0); + assertTrue(liveNode.containsKey("lastBlockReport")); // a. By default the upgrade domain isn't defined on any DN. // b. If the upgrade domain is set on a DN, JMX should have the same // value. From c571cda5c7d929477961dfff4176d7de4944d874 Mon Sep 17 00:00:00 2001 From: Mingliang Liu Date: Mon, 6 Mar 2017 16:53:30 -0800 Subject: [PATCH 018/188] HADOOP-14048. REDO operation of WASB#AtomicRename should create placeholder blob for destination folder. Contributed by NITIN VERMA --- .../apache/hadoop/fs/azure/NativeAzureFileSystem.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java index 0dfefaf196c..b1956a78541 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java @@ -566,6 +566,16 @@ public class NativeAzureFileSystem extends FileSystem { // Remove the source folder. Don't check explicitly if it exists, // to avoid triggering redo recursively. try { + // Rename the source folder 0-byte root file + // as destination folder 0-byte root file. + FileMetadata srcMetaData = this.getSourceMetadata(); + if (srcMetaData.getBlobMaterialization() == BlobMaterialization.Explicit) { + // We already have a lease. So let's just rename the source blob + // as destination blob under same lease. + fs.getStoreInterface().rename(this.getSrcKey(), this.getDstKey(), false, lease); + } + + // Now we can safely delete the source folder. fs.getStoreInterface().delete(srcKey, lease); } catch (Exception e) { LOG.info("Unable to delete source folder during folder rename redo. " From 52d7d5aa1a303cf70519a61487641211f4267c6f Mon Sep 17 00:00:00 2001 From: Mingliang Liu Date: Mon, 6 Mar 2017 17:10:11 -0800 Subject: [PATCH 019/188] Revert "HADOOP-13930. Azure: Add Authorization support to WASB. Contributed by Sivaguru Sankaridurg and Dushyanth" This reverts commit 6b7cd62b8cf12616b13142f2eb2cfc2f25796f0f. --- .../src/main/resources/core-default.xml | 10 - .../conf/TestCommonConfigurationFields.java | 2 - .../fs/azure/AzureNativeFileSystemStore.java | 4 +- .../fs/azure/NativeAzureFileSystem.java | 155 +--------- .../fs/azure/RemoteSASKeyGeneratorImpl.java | 179 ++++------- .../fs/azure/RemoteWasbAuthorizerImpl.java | 247 ---------------- .../fs/azure/SecureStorageInterfaceImpl.java | 6 +- .../fs/azure/WasbAuthorizationException.java | 40 --- .../fs/azure/WasbAuthorizationOperations.java | 44 --- .../fs/azure/WasbAuthorizerInterface.java | 47 --- .../hadoop/fs/azure/security/Constants.java | 54 ---- .../WasbDelegationTokenIdentifier.java | 48 --- .../fs/azure/security/WasbTokenRenewer.java | 124 -------- .../hadoop/fs/azure/security/package.html | 28 -- ...ache.hadoop.security.token.TokenIdentifier | 16 - ....apache.hadoop.security.token.TokenRenewer | 16 - .../hadoop-azure/src/site/markdown/index.md | 34 --- .../fs/azure/MockWasbAuthorizerImpl.java | 102 ------- ...estNativeAzureFileSystemAuthorization.java | 277 ------------------ 19 files changed, 62 insertions(+), 1371 deletions(-) delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationException.java delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationOperations.java delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizerInterface.java delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/Constants.java delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbDelegationTokenIdentifier.java delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbTokenRenewer.java delete mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/package.html delete mode 100644 hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier delete mode 100644 hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 52b58eddb1e..35be56bf81e 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -1292,16 +1292,6 @@ to specify the time (such as 2s, 2m, 1h, etc.). - - fs.azure.authorization - false - - Config flag to enable authorization support in WASB. Setting it to "true" enables - authorization support to WASB. Currently WASB authorization requires a remote service - to provide authorization that needs to be specified via fs.azure.authorization.remote.service.url - configuration - - diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java index 7410d297a09..966a8ac0d08 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java @@ -181,8 +181,6 @@ public class TestCommonConfigurationFields extends TestConfigurationFieldsBase { xmlPropsToSkipCompare.add("io.compression.codec.bzip2.library"); // - org.apache.hadoop.io.SequenceFile xmlPropsToSkipCompare.add("io.seqfile.local.dir"); - // - org.apache.hadoop.fs.azure.NativeAzureFileSystem - xmlPropsToSkipCompare.add("fs.azure.authorization"); } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java index 9d7ac80cdfd..07c389ce0c2 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java @@ -303,7 +303,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { private boolean useSecureMode = false; private boolean useLocalSasKeyMode = false; - + private String delegationToken; /** * A test hook interface that can modify the operation context we use for * Azure Storage operations, e.g. to inject errors. @@ -478,7 +478,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { this.storageInteractionLayer = new StorageInterfaceImpl(); } else { this.storageInteractionLayer = new SecureStorageInterfaceImpl( - useLocalSasKeyMode, conf); + useLocalSasKeyMode, conf, delegationToken); } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java index b1956a78541..4184a539418 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java @@ -25,12 +25,9 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.net.InetAddress; import java.net.URI; import java.net.URISyntaxException; -import java.net.URL; import java.nio.charset.Charset; -import java.security.PrivilegedExceptionAction; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; @@ -64,15 +61,10 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation; import org.apache.hadoop.fs.azure.metrics.AzureFileSystemMetricsSystem; -import org.apache.hadoop.fs.azure.security.Constants; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL; -import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator; -import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticator; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.Time; import org.slf4j.Logger; @@ -1114,39 +1106,7 @@ public class NativeAzureFileSystem extends FileSystem { // A counter to create unique (within-process) names for my metrics sources. private static AtomicInteger metricsSourceNameCounter = new AtomicInteger(); private boolean appendSupportEnabled = false; - private DelegationTokenAuthenticatedURL authURL; - private DelegationTokenAuthenticatedURL.Token authToken = new DelegationTokenAuthenticatedURL.Token(); - private String credServiceUrl; - - /** - * Configuration key to enable authorization support in WASB. - */ - public static final String KEY_AZURE_AUTHORIZATION = - "fs.azure.authorization"; - - /** - * Default value for the authorization support in WASB. - */ - private static final boolean DEFAULT_AZURE_AUTHORIZATION = false; - - /** - * Flag controlling authorization support in WASB. - */ - private boolean azureAuthorization = false; - - /** - * Flag controlling Kerberos support in WASB. - */ - private boolean kerberosSupportEnabled = false; - - /** - * Authorizer to use when authorization support is enabled in - * WASB. - */ - private WasbAuthorizerInterface authorizer = null; - - private String delegationToken = null; - + public NativeAzureFileSystem() { // set store in initialize() } @@ -1277,31 +1237,6 @@ public class NativeAzureFileSystem extends FileSystem { // Initialize thread counts from user configuration deleteThreadCount = conf.getInt(AZURE_DELETE_THREADS, DEFAULT_AZURE_DELETE_THREADS); renameThreadCount = conf.getInt(AZURE_RENAME_THREADS, DEFAULT_AZURE_RENAME_THREADS); - - this.azureAuthorization = conf.getBoolean(KEY_AZURE_AUTHORIZATION, - DEFAULT_AZURE_AUTHORIZATION); - this.kerberosSupportEnabled = conf.getBoolean( - Constants.AZURE_KERBEROS_SUPPORT_PROPERTY_NAME, false); - - if (this.azureAuthorization) { - - this.authorizer = - new RemoteWasbAuthorizerImpl(); - authorizer.init(conf); - } - if (UserGroupInformation.isSecurityEnabled() && kerberosSupportEnabled) { - DelegationTokenAuthenticator authenticator = new KerberosDelegationTokenAuthenticator(); - authURL = new DelegationTokenAuthenticatedURL(authenticator); - credServiceUrl = conf.get(Constants.KEY_CRED_SERVICE_URL, String - .format("http://%s:%s", - InetAddress.getLocalHost().getCanonicalHostName(), - Constants.DEFAULT_CRED_SERVICE_PORT)); - } - } - - @VisibleForTesting - public void updateWasbAuthorizer(WasbAuthorizerInterface authorizer) { - this.authorizer = authorizer; } private NativeFileSystemStore createDefaultStore(Configuration conf) { @@ -1415,15 +1350,6 @@ public class NativeAzureFileSystem extends FileSystem { return store; } - private void performAuthCheck(String path, String accessType, - String operation) throws WasbAuthorizationException, IOException { - - if (azureAuthorization && !this.authorizer.authorize(path, accessType)) { - throw new WasbAuthorizationException(operation - + " operation for Path : " + path + " not allowed"); - } - } - /** * Gets the metrics source for this file system. * This is mainly here for unit testing purposes. @@ -1446,10 +1372,6 @@ public class NativeAzureFileSystem extends FileSystem { LOG.debug("Opening file: {} for append", f); Path absolutePath = makeAbsolute(f); - - performAuthCheck(absolutePath.toString(), - WasbAuthorizationOperations.WRITE.toString(), "append"); - String key = pathToKey(absolutePath); FileMetadata meta = null; try { @@ -1650,10 +1572,6 @@ public class NativeAzureFileSystem extends FileSystem { } Path absolutePath = makeAbsolute(f); - - performAuthCheck(absolutePath.toString(), - WasbAuthorizationOperations.WRITE.toString(), "create"); - String key = pathToKey(absolutePath); FileMetadata existingMetadata = store.retrieveMetadata(key); @@ -1776,10 +1694,6 @@ public class NativeAzureFileSystem extends FileSystem { LOG.debug("Deleting file: {}", f.toString()); Path absolutePath = makeAbsolute(f); - - performAuthCheck(absolutePath.toString(), - WasbAuthorizationOperations.EXECUTE.toString(), "delete"); - String key = pathToKey(absolutePath); // Capture the metadata for the path. @@ -2050,10 +1964,6 @@ public class NativeAzureFileSystem extends FileSystem { // Capture the absolute path and the path to key. Path absolutePath = makeAbsolute(f); - - performAuthCheck(absolutePath.toString(), - WasbAuthorizationOperations.EXECUTE.toString(), "getFileStatus"); - String key = pathToKey(absolutePath); if (key.length() == 0) { // root always exists return newDirectory(null, absolutePath); @@ -2152,10 +2062,6 @@ public class NativeAzureFileSystem extends FileSystem { LOG.debug("Listing status for {}", f.toString()); Path absolutePath = makeAbsolute(f); - - performAuthCheck(absolutePath.toString(), - WasbAuthorizationOperations.EXECUTE.toString(), "list"); - String key = pathToKey(absolutePath); Set status = new TreeSet(); FileMetadata meta = null; @@ -2378,10 +2284,6 @@ public class NativeAzureFileSystem extends FileSystem { } Path absolutePath = makeAbsolute(f); - - performAuthCheck(absolutePath.toString(), - WasbAuthorizationOperations.EXECUTE.toString(), "mkdirs"); - PermissionStatus permissionStatus = null; if(noUmask) { // ensure owner still has wx permissions at the minimum @@ -2435,10 +2337,6 @@ public class NativeAzureFileSystem extends FileSystem { LOG.debug("Opening file: {}", f.toString()); Path absolutePath = makeAbsolute(f); - - performAuthCheck(absolutePath.toString(), - WasbAuthorizationOperations.READ.toString(), "read"); - String key = pathToKey(absolutePath); FileMetadata meta = null; try { @@ -2495,12 +2393,7 @@ public class NativeAzureFileSystem extends FileSystem { + " through WASB that has colons in the name"); } - Path absolutePath = makeAbsolute(src); - - performAuthCheck(absolutePath.toString(), - WasbAuthorizationOperations.EXECUTE.toString(), "rename"); - - String srcKey = pathToKey(absolutePath); + String srcKey = pathToKey(makeAbsolute(src)); if (srcKey.length() == 0) { // Cannot rename root of file system @@ -2802,10 +2695,6 @@ public class NativeAzureFileSystem extends FileSystem { @Override public void setPermission(Path p, FsPermission permission) throws FileNotFoundException, IOException { Path absolutePath = makeAbsolute(p); - - performAuthCheck(absolutePath.toString(), - WasbAuthorizationOperations.EXECUTE.toString(), "setPermission"); - String key = pathToKey(absolutePath); FileMetadata metadata = null; try { @@ -2844,10 +2733,6 @@ public class NativeAzureFileSystem extends FileSystem { public void setOwner(Path p, String username, String groupname) throws IOException { Path absolutePath = makeAbsolute(p); - - performAuthCheck(absolutePath.toString(), - WasbAuthorizationOperations.EXECUTE.toString(), "setOwner"); - String key = pathToKey(absolutePath); FileMetadata metadata = null; @@ -2910,42 +2795,6 @@ public class NativeAzureFileSystem extends FileSystem { isClosed = true; } - @Override - public Token getDelegationToken(final String renewer) throws IOException { - if(kerberosSupportEnabled) { - try { - final UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); - UserGroupInformation connectUgi = ugi.getRealUser(); - final UserGroupInformation proxyUser = connectUgi; - if (connectUgi == null) { - connectUgi = ugi; - } - if(!connectUgi.hasKerberosCredentials()){ - connectUgi = UserGroupInformation.getLoginUser(); - } - connectUgi.checkTGTAndReloginFromKeytab(); - return connectUgi.doAs(new PrivilegedExceptionAction>() { - @Override - public Token run() throws Exception { - return authURL.getDelegationToken(new URL(credServiceUrl - + Constants.DEFAULT_DELEGATION_TOKEN_MANAGER_ENDPOINT), - authToken, renewer, (proxyUser != null)? ugi.getShortUserName(): null); - } - }); - } catch (Exception ex) { - LOG.error("Error in fetching the delegation token from remote service", - ex); - if (ex instanceof IOException) { - throw (IOException) ex; - } else { - throw new IOException(ex); - } - } - } else { - return super.getDelegationToken(renewer); - } - } - /** * A handler that defines what to do with blobs whose upload was * interrupted. diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteSASKeyGeneratorImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteSASKeyGeneratorImpl.java index 43672b27a66..404419d7422 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteSASKeyGeneratorImpl.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteSASKeyGeneratorImpl.java @@ -19,23 +19,10 @@ package org.apache.hadoop.fs.azure; import java.io.IOException; -import java.net.InetAddress; import java.net.URI; import java.net.URISyntaxException; -import java.net.UnknownHostException; -import java.security.PrivilegedExceptionAction; -import java.util.Iterator; -import org.apache.commons.lang.Validate; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.azure.security.Constants; -import org.apache.hadoop.fs.azure.security.WasbDelegationTokenIdentifier; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.authentication.client.AuthenticatedURL; -import org.apache.hadoop.security.authentication.client.Authenticator; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.TokenIdentifier; -import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticator; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.utils.URIBuilder; import org.slf4j.Logger; @@ -56,6 +43,12 @@ public class RemoteSASKeyGeneratorImpl extends SASKeyGeneratorImpl { public static final Logger LOG = LoggerFactory.getLogger(AzureNativeFileSystemStore.class); + /** + * Configuration parameter name expected in the Configuration + * object to provide the url of the remote service {@value} + */ + private static final String KEY_CRED_SERVICE_URL = + "fs.azure.cred.service.url"; /** * Container SAS Key generation OP name. {@value} @@ -89,7 +82,7 @@ public class RemoteSASKeyGeneratorImpl extends SASKeyGeneratorImpl { * Query parameter name for user info {@value} */ private static final String DELEGATION_TOKEN_QUERY_PARAM_NAME = - "delegation"; + "delegation_token"; /** * Query parameter name for the relative path inside the storage @@ -101,40 +94,24 @@ public class RemoteSASKeyGeneratorImpl extends SASKeyGeneratorImpl { private String delegationToken = ""; private String credServiceUrl = ""; private WasbRemoteCallHelper remoteCallHelper = null; - private boolean isSecurityEnabled; - private boolean isKerberosSupportEnabled; public RemoteSASKeyGeneratorImpl(Configuration conf) { super(conf); } - public boolean initialize(Configuration conf) { + public boolean initialize(Configuration conf, String delegationToken) { LOG.debug("Initializing RemoteSASKeyGeneratorImpl instance"); - Iterator> tokenIterator = null; - try { - tokenIterator = UserGroupInformation.getCurrentUser().getCredentials() - .getAllTokens().iterator(); - while (tokenIterator.hasNext()) { - Token iteratedToken = tokenIterator.next(); - if (iteratedToken.getKind().equals(WasbDelegationTokenIdentifier.TOKEN_KIND)) { - delegationToken = iteratedToken.encodeToUrlString(); - } - } - } catch (IOException e) { - LOG.error("Error in fetching the WASB delegation token"); - } + credServiceUrl = conf.get(KEY_CRED_SERVICE_URL); - try { - credServiceUrl = conf.get(Constants.KEY_CRED_SERVICE_URL, String - .format("http://%s:%s", - InetAddress.getLocalHost().getCanonicalHostName(), - Constants.DEFAULT_CRED_SERVICE_PORT)); - } catch (UnknownHostException e) { - LOG.error("Invalid CredService Url, configure it correctly."); + if (delegationToken == null || delegationToken.isEmpty()) { + LOG.error("Delegation Token not provided for initialization" + + " of RemoteSASKeyGenerator"); return false; } + this.delegationToken = delegationToken; + if (credServiceUrl == null || credServiceUrl.isEmpty()) { LOG.error("CredService Url not found in configuration to initialize" + " RemoteSASKeyGenerator"); @@ -142,17 +119,16 @@ public class RemoteSASKeyGeneratorImpl extends SASKeyGeneratorImpl { } remoteCallHelper = new WasbRemoteCallHelper(); - this.isSecurityEnabled = UserGroupInformation.isSecurityEnabled(); - this.isKerberosSupportEnabled = conf.getBoolean( - Constants.AZURE_KERBEROS_SUPPORT_PROPERTY_NAME, false); - LOG.debug("Initialization of RemoteSASKeyGenerator instance successful"); + LOG.debug("Initialization of RemoteSASKeyGenerator instance successfull"); return true; } @Override public URI getContainerSASUri(String storageAccount, String container) throws SASKeyGenerationException { + try { + LOG.debug("Generating Container SAS Key for Container {} " + "inside Storage Account {} ", container, storageAccount); URIBuilder uriBuilder = new URIBuilder(credServiceUrl); @@ -163,131 +139,84 @@ public class RemoteSASKeyGeneratorImpl extends SASKeyGeneratorImpl { container); uriBuilder.addParameter(SAS_EXPIRY_QUERY_PARAM_NAME, "" + getSasKeyExpiryPeriod()); - if (isSecurityEnabled && (delegationToken != null && !delegationToken - .isEmpty())) { - uriBuilder.addParameter(DELEGATION_TOKEN_QUERY_PARAM_NAME, - this.delegationToken); - } + uriBuilder.addParameter(DELEGATION_TOKEN_QUERY_PARAM_NAME, + this.delegationToken); - UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); - UserGroupInformation connectUgi = ugi.getRealUser(); - if (connectUgi == null) { - connectUgi = ugi; + RemoteSASKeyGenerationResponse sasKeyResponse = + makeRemoteRequest(uriBuilder.build()); + + if (sasKeyResponse == null) { + throw new SASKeyGenerationException("RemoteSASKeyGenerationResponse" + + " object null from remote call"); + } else if (sasKeyResponse.getResponseCode() + == REMOTE_CALL_SUCCESS_CODE) { + return new URI(sasKeyResponse.getSasKey()); } else { - uriBuilder.addParameter(Constants.DOAS_PARAM, ugi.getShortUserName()); + throw new SASKeyGenerationException("Remote Service encountered error" + + " in SAS Key generation : " + + sasKeyResponse.getResponseMessage()); } - - if(isSecurityEnabled && !connectUgi.hasKerberosCredentials()){ - connectUgi = UserGroupInformation.getLoginUser(); - } - return getSASKey(uriBuilder.build(), connectUgi); } catch (URISyntaxException uriSyntaxEx) { throw new SASKeyGenerationException("Encountered URISyntaxException " + "while building the HttpGetRequest to remote cred service", uriSyntaxEx); - } catch (IOException e) { - throw new SASKeyGenerationException("Encountered IOException" - + " while building the HttpGetRequest to remote service", e); } } @Override public URI getRelativeBlobSASUri(String storageAccount, String container, String relativePath) throws SASKeyGenerationException { + try { + LOG.debug("Generating RelativePath SAS Key for relativePath {} inside" + " Container {} inside Storage Account {} ", relativePath, container, storageAccount); URIBuilder uriBuilder = new URIBuilder(credServiceUrl); uriBuilder.setPath("/" + BLOB_SAS_OP); - uriBuilder.addParameter(STORAGE_ACCOUNT_QUERY_PARAM_NAME, storageAccount); - uriBuilder.addParameter(CONTAINER_QUERY_PARAM_NAME, container); + uriBuilder.addParameter(STORAGE_ACCOUNT_QUERY_PARAM_NAME, + storageAccount); + uriBuilder.addParameter(CONTAINER_QUERY_PARAM_NAME, + container); uriBuilder.addParameter(RELATIVE_PATH_QUERY_PARAM_NAME, relativePath); uriBuilder.addParameter(SAS_EXPIRY_QUERY_PARAM_NAME, "" + getSasKeyExpiryPeriod()); + uriBuilder.addParameter(DELEGATION_TOKEN_QUERY_PARAM_NAME, + this.delegationToken); - if (isSecurityEnabled && (delegationToken != null && !delegationToken - .isEmpty())) { - uriBuilder.addParameter(DELEGATION_TOKEN_QUERY_PARAM_NAME, - this.delegationToken); + RemoteSASKeyGenerationResponse sasKeyResponse = + makeRemoteRequest(uriBuilder.build()); + + if (sasKeyResponse == null) { + throw new SASKeyGenerationException("RemoteSASKeyGenerationResponse" + + " object null from remote call"); + } else if (sasKeyResponse.getResponseCode() + == REMOTE_CALL_SUCCESS_CODE) { + return new URI(sasKeyResponse.getSasKey()); + } else { + throw new SASKeyGenerationException("Remote Service encountered error" + + " in SAS Key generation : " + + sasKeyResponse.getResponseMessage()); } - - UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); - UserGroupInformation connectUgi = ugi.getRealUser(); - if (connectUgi == null) { - connectUgi = ugi; - } else{ - uriBuilder.addParameter(Constants.DOAS_PARAM, ugi.getShortUserName()); - } - - if(isSecurityEnabled && !connectUgi.hasKerberosCredentials()){ - connectUgi = UserGroupInformation.getLoginUser(); - } - return getSASKey(uriBuilder.build(), connectUgi); } catch (URISyntaxException uriSyntaxEx) { throw new SASKeyGenerationException("Encountered URISyntaxException" + " while building the HttpGetRequest to " + " remote service", uriSyntaxEx); - } catch (IOException e) { - throw new SASKeyGenerationException("Encountered IOException" - + " while building the HttpGetRequest to remote service", e); - } - } - - private URI getSASKey(final URI uri, UserGroupInformation connectUgi) - throws URISyntaxException, SASKeyGenerationException { - RemoteSASKeyGenerationResponse sasKeyResponse = null; - try { - connectUgi.checkTGTAndReloginFromKeytab(); - sasKeyResponse = connectUgi.doAs(new PrivilegedExceptionAction() { - @Override - public RemoteSASKeyGenerationResponse run() throws Exception { - AuthenticatedURL.Token token = null; - if (isKerberosSupportEnabled && UserGroupInformation.isSecurityEnabled() && ( - delegationToken == null || delegationToken.isEmpty())) { - token = new AuthenticatedURL.Token(); - final Authenticator kerberosAuthenticator = new KerberosDelegationTokenAuthenticator(); - kerberosAuthenticator.authenticate(uri.toURL(), token); - Validate.isTrue(token.isSet(), - "Authenticated Token is NOT present. The request cannot proceed."); - } - return makeRemoteRequest(uri, (token != null ? token.toString() : null)); - } - }); - } catch (InterruptedException e) { - LOG.error("Error fetching the SAS Key from Remote Service", e); - } catch (IOException e) { - LOG.error("Error fetching the SAS Key from Remote Service", e); - } - - if (sasKeyResponse == null) { - throw new SASKeyGenerationException( - "RemoteSASKeyGenerationResponse" + " object null from remote call"); - } else if (sasKeyResponse.getResponseCode() == REMOTE_CALL_SUCCESS_CODE) { - return new URI(sasKeyResponse.getSasKey()); - } else { - throw new SASKeyGenerationException("Remote Service encountered error" - + " in SAS Key generation : " + sasKeyResponse.getResponseMessage()); } } /** * Helper method to make a remote request. * @param uri - Uri to use for the remote request - * @param token - hadoop.auth token for the remote request * @return RemoteSASKeyGenerationResponse */ - private RemoteSASKeyGenerationResponse makeRemoteRequest(URI uri, String token) + private RemoteSASKeyGenerationResponse makeRemoteRequest(URI uri) throws SASKeyGenerationException { try { - HttpGet httpGet = new HttpGet(uri); - if(token != null){ - httpGet.setHeader("Cookie", AuthenticatedURL.AUTH_COOKIE + "=" + token); - } String responseBody = - remoteCallHelper.makeRemoteGetRequest(httpGet); + remoteCallHelper.makeRemoteGetRequest(new HttpGet(uri)); ObjectMapper objectMapper = new ObjectMapper(); return objectMapper.readValue(responseBody, diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java deleted file mode 100644 index bb1093d3ff2..00000000000 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java +++ /dev/null @@ -1,247 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure; - -import java.io.IOException; -import java.net.InetAddress; -import java.net.URISyntaxException; -import java.security.PrivilegedExceptionAction; -import java.util.Iterator; - -import org.apache.commons.lang.Validate; -import org.apache.hadoop.conf.Configuration; - -import org.apache.hadoop.fs.azure.security.Constants; -import org.apache.hadoop.fs.azure.security.WasbDelegationTokenIdentifier; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.authentication.client.AuthenticatedURL; -import org.apache.hadoop.security.authentication.client.Authenticator; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.TokenIdentifier; -import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticator; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.utils.URIBuilder; - -import com.fasterxml.jackson.core.JsonParseException; -import org.codehaus.jackson.map.JsonMappingException; -import org.codehaus.jackson.map.ObjectMapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.hadoop.fs.azure.WasbRemoteCallHelper.REMOTE_CALL_SUCCESS_CODE; - -/** - * Class implementing WasbAuthorizerInterface using a remote - * service that implements the authorization operation. This - * class expects the url of the remote service to be passed - * via config. - */ -public class RemoteWasbAuthorizerImpl implements WasbAuthorizerInterface { - - public static final Logger LOG = - LoggerFactory.getLogger(RemoteWasbAuthorizerImpl.class); - private String remoteAuthorizerServiceUrl = ""; - - /** - * Configuration parameter name expected in the Configuration object to - * provide the url of the remote service. {@value} - */ - public static final String KEY_REMOTE_AUTH_SERVICE_URL = - "fs.azure.authorization.remote.service.url"; - - /** - * Authorization operation OP name in the remote service {@value} - */ - private static final String CHECK_AUTHORIZATION_OP = - "CHECK_AUTHORIZATION"; - - /** - * Query parameter specifying the access operation type. {@value} - */ - private static final String ACCESS_OPERATION_QUERY_PARAM_NAME = - "operation_type"; - - /** - * Query parameter specifying the wasb absolute path. {@value} - */ - private static final String WASB_ABSOLUTE_PATH_QUERY_PARAM_NAME = - "wasb_absolute_path"; - - /** - * Query parameter name for user info {@value} - */ - private static final String DELEGATION_TOKEN_QUERY_PARAM_NAME = - "delegation"; - - private WasbRemoteCallHelper remoteCallHelper = null; - private String delegationToken; - private boolean isSecurityEnabled; - private boolean isKerberosSupportEnabled; - - @Override - public void init(Configuration conf) - throws WasbAuthorizationException, IOException { - LOG.debug("Initializing RemoteWasbAuthorizerImpl instance"); - delegationToken = UserGroupInformation.getCurrentUser().getCredentials().getToken(WasbDelegationTokenIdentifier.TOKEN_KIND).encodeToUrlString(); - - remoteAuthorizerServiceUrl = conf.get(KEY_REMOTE_AUTH_SERVICE_URL, String - .format("http://%s:%s", - InetAddress.getLocalHost().getCanonicalHostName(), - Constants.DEFAULT_CRED_SERVICE_PORT)); - - if (remoteAuthorizerServiceUrl == null - || remoteAuthorizerServiceUrl.isEmpty()) { - throw new WasbAuthorizationException( - "fs.azure.authorization.remote.service.url config not set" - + " in configuration."); - } - - this.remoteCallHelper = new WasbRemoteCallHelper(); - this.isSecurityEnabled = UserGroupInformation.isSecurityEnabled(); - this.isKerberosSupportEnabled = conf.getBoolean( - Constants.AZURE_KERBEROS_SUPPORT_PROPERTY_NAME, false); - } - - @Override - public boolean authorize(String wasbAbsolutePath, String accessType) - throws WasbAuthorizationException, IOException { - try { - final URIBuilder uriBuilder = new URIBuilder(remoteAuthorizerServiceUrl); - uriBuilder.setPath("/" + CHECK_AUTHORIZATION_OP); - uriBuilder.addParameter(WASB_ABSOLUTE_PATH_QUERY_PARAM_NAME, - wasbAbsolutePath); - uriBuilder.addParameter(ACCESS_OPERATION_QUERY_PARAM_NAME, - accessType); - if (isSecurityEnabled && (delegationToken != null && !delegationToken - .isEmpty())) { - uriBuilder - .addParameter(DELEGATION_TOKEN_QUERY_PARAM_NAME, delegationToken); - } - String responseBody = null; - UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); - UserGroupInformation connectUgi = ugi.getRealUser(); - if (connectUgi == null) { - connectUgi = ugi; - } else{ - uriBuilder.addParameter(Constants.DOAS_PARAM, ugi.getShortUserName()); - } - if(isSecurityEnabled && !connectUgi.hasKerberosCredentials()){ - connectUgi = UserGroupInformation.getLoginUser(); - } - connectUgi.checkTGTAndReloginFromKeytab(); - try { - responseBody = connectUgi.doAs(new PrivilegedExceptionAction(){ - @Override - public String run() throws Exception { - AuthenticatedURL.Token token = null; - HttpGet httpGet = new HttpGet(uriBuilder.build()); - if (isKerberosSupportEnabled && UserGroupInformation.isSecurityEnabled() && ( - delegationToken == null || delegationToken.isEmpty())) { - token = new AuthenticatedURL.Token(); - final Authenticator kerberosAuthenticator = new KerberosDelegationTokenAuthenticator(); - kerberosAuthenticator - .authenticate(uriBuilder.build().toURL(), token); - Validate.isTrue(token.isSet(), - "Authenticated Token is NOT present. The request cannot proceed."); - if(token != null){ - httpGet.setHeader("Cookie", AuthenticatedURL.AUTH_COOKIE + "=" + token); - } - } - return remoteCallHelper.makeRemoteGetRequest(httpGet); - }}); - } catch (InterruptedException e) { - LOG.error("Error in check authorization", e); - } - - ObjectMapper objectMapper = new ObjectMapper(); - RemoteAuthorizerResponse authorizerResponse = - objectMapper.readValue(responseBody, RemoteAuthorizerResponse.class); - - if (authorizerResponse == null) { - throw new WasbAuthorizationException( - "RemoteAuthorizerResponse object null from remote call"); - } else if (authorizerResponse.getResponseCode() - == REMOTE_CALL_SUCCESS_CODE) { - return authorizerResponse.getAuthorizationResult(); - } else { - throw new WasbAuthorizationException("Remote authorization" - + " serivce encountered an error " - + authorizerResponse.getResponseMessage()); - } - } catch (URISyntaxException | WasbRemoteCallException - | JsonParseException | JsonMappingException ex) { - throw new WasbAuthorizationException(ex); - } - } -} - -/** - * POJO representing the response expected from a remote - * authorization service. - * The remote service is expected to return the authorization - * response in the following JSON format - * { - * "responseCode" : 0 or non-zero , - * "responseMessage" : relavant message of failure - * "authorizationResult" : authorization result - * true - if auhorization allowed - * false - otherwise. - * - * } - */ -class RemoteAuthorizerResponse { - - private int responseCode; - private boolean authorizationResult; - private String responseMessage; - - public RemoteAuthorizerResponse(){ - } - - public RemoteAuthorizerResponse(int responseCode, - boolean authorizationResult, String message) { - this.responseCode = responseCode; - this.authorizationResult = authorizationResult; - this.responseMessage = message; - } - - public int getResponseCode() { - return responseCode; - } - - public void setResponseCode(int responseCode) { - this.responseCode = responseCode; - } - - public boolean getAuthorizationResult() { - return authorizationResult; - } - - public void setAuthorizationResult(boolean authorizationResult) { - this.authorizationResult = authorizationResult; - } - - public String getResponseMessage() { - return responseMessage; - } - - public void setResponseMessage(String message) { - this.responseMessage = message; - } -} \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java index 5ec9136a9a5..6749a76f7f4 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java @@ -69,17 +69,19 @@ public class SecureStorageInterfaceImpl extends StorageInterface { public static final String SAS_ERROR_CODE = "SAS Error"; private SASKeyGeneratorInterface sasKeyGenerator; private String storageAccount; + private String delegationToken; public SecureStorageInterfaceImpl(boolean useLocalSASKeyMode, - Configuration conf) + Configuration conf, String delegationToken) throws SecureModeException { + this.delegationToken = delegationToken; if (useLocalSASKeyMode) { this.sasKeyGenerator = new LocalSASKeyGeneratorImpl(conf); } else { RemoteSASKeyGeneratorImpl remoteSasKeyGenerator = new RemoteSASKeyGeneratorImpl(conf); - if (!remoteSasKeyGenerator.initialize(conf)) { + if (!remoteSasKeyGenerator.initialize(conf, this.delegationToken)) { throw new SecureModeException("Remote SAS Key mode could" + " not be initialized"); } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationException.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationException.java deleted file mode 100644 index eff9248dffe..00000000000 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationException.java +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure; - -/** - * Exception that gets thrown during the authorization failures - * in WASB. - */ -public class WasbAuthorizationException extends AzureException { - - private static final long serialVersionUID = 1L; - - public WasbAuthorizationException(String message) { - super(message); - } - - public WasbAuthorizationException(String message, Throwable cause) { - super(message, cause); - } - - public WasbAuthorizationException(Throwable t) { - super(t); - } -} \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationOperations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationOperations.java deleted file mode 100644 index bd768372215..00000000000 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationOperations.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure; - -/** - * Different authorization operations supported - * in WASB. - */ - -public enum WasbAuthorizationOperations { - - READ, WRITE, EXECUTE; - - @Override - public String toString() { - switch(this) { - case READ: - return "read"; - case WRITE: - return "write"; - case EXECUTE: - return "execute"; - default: - throw new IllegalArgumentException( - "Invalid Auhtorization Operation"); - } - } -} \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizerInterface.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizerInterface.java deleted file mode 100644 index 0c61997a61e..00000000000 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizerInterface.java +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure; - -import java.io.IOException; - -import org.apache.hadoop.conf.Configuration; - -/** - * Interface to implement authorization support in WASB. - * API's of this interface will be implemented in the - * StorageInterface Layer before making calls to Azure - * Storage. - */ -public interface WasbAuthorizerInterface { - /** - * Initializer method - * @param conf - Configuration object - */ - public void init(Configuration conf) - throws WasbAuthorizationException, IOException; - - /** - * Authorizer API to authorize access in WASB. - * @param wasbAbsolutePath : Absolute WASB Path used for access. - * @param accessType : Type of access - * @return : true - If access allowed false - If access is not allowed. - */ - public boolean authorize(String wasbAbsolutePath, String accessType) - throws WasbAuthorizationException, IOException; -} \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/Constants.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/Constants.java deleted file mode 100644 index 105068710d5..00000000000 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/Constants.java +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure.security; - -/** - * Constants for used with WASB security implementation. - */ -public final class Constants { - - private Constants() { - } - - /** - * Configuration parameter name expected in the Configuration - * object to provide the url of the remote service {@value} - */ - public static final String KEY_CRED_SERVICE_URL = "fs.azure.cred.service.url"; - /** - * Default port of the remote service used as delegation token manager and Azure storage SAS key generator. - */ - public static final int DEFAULT_CRED_SERVICE_PORT = 50911; - - /** - * Default remote delegation token manager endpoint. - */ - public static final String DEFAULT_DELEGATION_TOKEN_MANAGER_ENDPOINT = "/tokenmanager/v1"; - - /** - * The configuration property to enable Kerberos support. - */ - - public static final String AZURE_KERBEROS_SUPPORT_PROPERTY_NAME = "fs.azure.enable.kerberos.support"; - - /** - * Parameter to be used for impersonation. - */ - public static final String DOAS_PARAM="doas"; -} diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbDelegationTokenIdentifier.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbDelegationTokenIdentifier.java deleted file mode 100644 index 530e04572e2..00000000000 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbDelegationTokenIdentifier.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure.security; - -import org.apache.hadoop.io.Text; -import org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier; - -/** - * Delegation token Identifier for WASB delegation tokens. - */ -public class WasbDelegationTokenIdentifier extends DelegationTokenIdentifier { - public static final Text TOKEN_KIND = new Text("WASB delegation"); - - public WasbDelegationTokenIdentifier(){ - super(TOKEN_KIND); - } - - public WasbDelegationTokenIdentifier(Text kind) { - super(kind); - } - - public WasbDelegationTokenIdentifier(Text kind, Text owner, Text renewer, - Text realUser) { - super(kind, owner, renewer, realUser); - } - - @Override - public Text getKind() { - return TOKEN_KIND; - } - -} diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbTokenRenewer.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbTokenRenewer.java deleted file mode 100644 index 09b7349e120..00000000000 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbTokenRenewer.java +++ /dev/null @@ -1,124 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure.security; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.TokenRenewer; -import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; -import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL; -import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator; -import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticator; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.InetAddress; -import java.net.URL; -import java.security.PrivilegedExceptionAction; - -/** - * Token Renewer for renewing WASB delegation tokens with remote service. - */ -public class WasbTokenRenewer extends TokenRenewer { - public static final Logger LOG = LoggerFactory - .getLogger(WasbTokenRenewer.class); - - @Override - public boolean handleKind(Text kind) { - return WasbDelegationTokenIdentifier.TOKEN_KIND.equals(kind); - } - - @Override - public boolean isManaged(Token token) throws IOException { - return true; - } - - @Override - public long renew(final Token token, Configuration conf) - throws IOException, InterruptedException { - LOG.debug("Renewing the delegation token"); - final UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); - UserGroupInformation connectUgi = ugi.getRealUser(); - final UserGroupInformation proxyUser = connectUgi; - if (connectUgi == null) { - connectUgi = ugi; - } - if(!connectUgi.hasKerberosCredentials()){ - connectUgi = UserGroupInformation.getLoginUser(); - } - connectUgi.checkTGTAndReloginFromKeytab(); - final DelegationTokenAuthenticatedURL.Token authToken = new DelegationTokenAuthenticatedURL.Token(); - authToken - .setDelegationToken((Token) token); - final String credServiceUrl = conf.get(Constants.KEY_CRED_SERVICE_URL, - String.format("http://%s:%s", - InetAddress.getLocalHost().getCanonicalHostName(), - Constants.DEFAULT_CRED_SERVICE_PORT)); - DelegationTokenAuthenticator authenticator = new KerberosDelegationTokenAuthenticator(); - final DelegationTokenAuthenticatedURL authURL = new DelegationTokenAuthenticatedURL( - authenticator); - - return connectUgi.doAs(new PrivilegedExceptionAction() { - @Override - public Long run() throws Exception { - return authURL.renewDelegationToken(new URL(credServiceUrl - + Constants.DEFAULT_DELEGATION_TOKEN_MANAGER_ENDPOINT), - authToken, (proxyUser != null) ? ugi.getShortUserName() : null); - } - }); - } - - @Override - public void cancel(final Token token, Configuration conf) - throws IOException, InterruptedException { - LOG.debug("Cancelling the delegation token"); - final UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); - UserGroupInformation connectUgi = ugi.getRealUser(); - final UserGroupInformation proxyUser = connectUgi; - if (connectUgi == null) { - connectUgi = ugi; - } - if(!connectUgi.hasKerberosCredentials()){ - connectUgi = UserGroupInformation.getLoginUser(); - } - connectUgi.checkTGTAndReloginFromKeytab(); - final DelegationTokenAuthenticatedURL.Token authToken = new DelegationTokenAuthenticatedURL.Token(); - authToken - .setDelegationToken((Token) token); - final String credServiceUrl = conf.get(Constants.KEY_CRED_SERVICE_URL, - String.format("http://%s:%s", - InetAddress.getLocalHost().getCanonicalHostName(), - Constants.DEFAULT_CRED_SERVICE_PORT)); - DelegationTokenAuthenticator authenticator = new KerberosDelegationTokenAuthenticator(); - final DelegationTokenAuthenticatedURL authURL = new DelegationTokenAuthenticatedURL( - authenticator); - connectUgi.doAs(new PrivilegedExceptionAction() { - @Override - public Void run() throws Exception { - authURL.cancelDelegationToken(new URL(credServiceUrl - + Constants.DEFAULT_DELEGATION_TOKEN_MANAGER_ENDPOINT), - authToken, (proxyUser != null) ? ugi.getShortUserName() : null); - return null; - } - }); - } -} \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/package.html b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/package.html deleted file mode 100644 index fe58c0a272a..00000000000 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/package.html +++ /dev/null @@ -1,28 +0,0 @@ - - - - - - -

- Infrastructure for WASB client Security to work with Kerberos and Delegation - tokens. -

- - - diff --git a/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier b/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier deleted file mode 100644 index 7ec8216deb0..00000000000 --- a/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -org.apache.hadoop.fs.azure.security.WasbDelegationTokenIdentifier \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer b/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer deleted file mode 100644 index f9c590aad8d..00000000000 --- a/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -org.apache.hadoop.fs.azure.security.WasbTokenRenewer \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/index.md b/hadoop-tools/hadoop-azure/src/site/markdown/index.md index a06d297ea7c..2865223acdb 100644 --- a/hadoop-tools/hadoop-azure/src/site/markdown/index.md +++ b/hadoop-tools/hadoop-azure/src/site/markdown/index.md @@ -333,40 +333,6 @@ The service is expected to return a response in JSON format: "sasKey" : Requested SAS Key } ``` - -## Authorization Support in WASB. - -Authorization support can be enabled in WASB using the following configuration: - -``` - - fs.azure.authorization - true - -``` - The current implementation of authorization relies on the presence of an external service that can enforce - the authorization. The service is expected to be running on a URL provided by the following config. - -``` - - fs.azure.authorization.remote.service.url - {URL} - -``` - - The remote service is expected to provide support for the following REST call: ```{URL}/CHECK_AUTHORIZATION``` - An example request: - ```{URL}/CHECK_AUTHORIZATION?wasb_absolute_path=&operation_type=&delegation_token=``` - - The service is expected to return a response in JSON format: - ``` - { - "responseCode" : 0 or non-zero , - "responseMessage" : relavant message on failure , - "authorizationResult" : true/false - } - ``` - ## Testing the hadoop-azure Module The hadoop-azure module includes a full suite of unit tests. Most of the tests diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java deleted file mode 100644 index af5a537ce4d..00000000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java +++ /dev/null @@ -1,102 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure; - -import java.util.HashMap; -import java.util.Map; - -import org.apache.hadoop.conf.Configuration; - -/** - * A mock wasb authorizer implementation. - */ - -public class MockWasbAuthorizerImpl implements WasbAuthorizerInterface { - - private Map authRules; - - @Override - public void init(Configuration conf) { - authRules = new HashMap(); - } - - public void addAuthRule(String wasbAbsolutePath, - String accessType, boolean access) { - AuthorizationComponent component = - new AuthorizationComponent(wasbAbsolutePath, accessType); - this.authRules.put(component, access); - } - - @Override - public boolean authorize(String wasbAbsolutePath, String accessType) - throws WasbAuthorizationException { - - AuthorizationComponent component = - new AuthorizationComponent(wasbAbsolutePath, accessType); - - if (authRules.containsKey(component)) { - return authRules.get(component); - } else { - return false; - } - } -} - -class AuthorizationComponent { - - private String wasbAbsolutePath; - private String accessType; - - public AuthorizationComponent(String wasbAbsolutePath, - String accessType) { - this.wasbAbsolutePath = wasbAbsolutePath; - this.accessType = accessType; - } - - @Override - public int hashCode() { - return this.wasbAbsolutePath.hashCode() ^ this.accessType.hashCode(); - } - - @Override - public boolean equals(Object obj) { - - if (obj == this) { - return true; - } - - if (obj == null - || !(obj instanceof AuthorizationComponent)) { - return false; - } - - return ((AuthorizationComponent)obj). - getWasbAbsolutePath().equals(this.wasbAbsolutePath) - && ((AuthorizationComponent)obj). - getAccessType().equals(this.accessType); - } - - public String getWasbAbsolutePath() { - return this.wasbAbsolutePath; - } - - public String getAccessType() { - return accessType; - } -} \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java deleted file mode 100644 index e76533550b4..00000000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java +++ /dev/null @@ -1,277 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.Path; -import org.junit.Test; - -import com.sun.tools.javac.util.Assert; - -/** - * Test class to hold all WASB authorization tests. - */ -public class TestNativeAzureFileSystemAuthorization - extends AbstractWasbTestBase { - - @Override - protected AzureBlobStorageTestAccount createTestAccount() throws Exception { - Configuration conf = new Configuration(); - conf.set(NativeAzureFileSystem.KEY_AZURE_AUTHORIZATION, "true"); - conf.set(RemoteWasbAuthorizerImpl.KEY_REMOTE_AUTH_SERVICE_URL, "test_url"); - return AzureBlobStorageTestAccount.create(conf); - } - - /** - * Positive test to verify Create and delete access check - * @throws Throwable - */ - @Test - public void testCreateAccessCheckPositive() throws Throwable { - - AzureBlobStorageTestAccount testAccount = createTestAccount(); - NativeAzureFileSystem fs = testAccount.getFileSystem(); - - String testFile = "test.dat"; - Path testPath = new Path(fs.getWorkingDirectory(), testFile); - - MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); - authorizer.init(null); - authorizer.addAuthRule(testPath.toString(), - WasbAuthorizationOperations.WRITE.toString(), true); - authorizer.addAuthRule(testPath.toString(), - WasbAuthorizationOperations.EXECUTE.toString(), true); - fs.updateWasbAuthorizer(authorizer); - authorizer.addAuthRule(fs.getWorkingDirectory().toString(), - WasbAuthorizationOperations.EXECUTE.toString(), true); - - fs.create(testPath); - Assert.check(fs.exists(testPath)); - fs.delete(testPath, false); - } - - /** - * Negative test to verify Create access check - * @throws Throwable - */ - - @Test(expected=WasbAuthorizationException.class) - public void testCreateAccessCheckNegative() throws Throwable { - - AzureBlobStorageTestAccount testAccount = createTestAccount(); - NativeAzureFileSystem fs = testAccount.getFileSystem(); - - String testFile = "test.dat"; - Path testPath = new Path(fs.getWorkingDirectory(), testFile); - - MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); - authorizer.init(null); - authorizer.addAuthRule(testPath.toString(), - WasbAuthorizationOperations.WRITE.toString(), false); - fs.updateWasbAuthorizer(authorizer); - - fs.create(new Path(testFile)); - } - - /** - * Positive test to verify Create and delete access check - * @throws Throwable - */ - @Test - public void testListAccessCheckPositive() throws Throwable { - - AzureBlobStorageTestAccount testAccount = createTestAccount(); - NativeAzureFileSystem fs = testAccount.getFileSystem(); - - String testFolder = "\\"; - Path testPath = new Path(fs.getWorkingDirectory(), testFolder); - - MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); - authorizer.init(null); - authorizer.addAuthRule(testPath.toString(), - WasbAuthorizationOperations.EXECUTE.toString(), true); - fs.updateWasbAuthorizer(authorizer); - - fs.listStatus(testPath); - } - - /** - * Negative test to verify Create access check - * @throws Throwable - */ - - @Test(expected=WasbAuthorizationException.class) - public void testListAccessCheckNegative() throws Throwable { - - AzureBlobStorageTestAccount testAccount = createTestAccount(); - NativeAzureFileSystem fs = testAccount.getFileSystem(); - - String testFolder = "\\"; - Path testPath = new Path(fs.getWorkingDirectory(), testFolder); - - MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); - authorizer.init(null); - authorizer.addAuthRule(testPath.toString(), - WasbAuthorizationOperations.EXECUTE.toString(), false); - fs.updateWasbAuthorizer(authorizer); - - fs.listStatus(testPath); - } - - /** - * Positive test to verify rename access check. - * @throws Throwable - */ - @Test - public void testRenameAccessCheckPositive() throws Throwable { - - AzureBlobStorageTestAccount testAccount = createTestAccount(); - NativeAzureFileSystem fs = testAccount.getFileSystem(); - - String testFile = "test.dat"; - Path testPath = new Path(fs.getWorkingDirectory(), testFile); - String renameFile = "test2.dat"; - Path renamePath = new Path(fs.getWorkingDirectory(), renameFile); - - MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); - authorizer.init(null); - authorizer.addAuthRule(testPath.toString(), - WasbAuthorizationOperations.WRITE.toString(), true); - authorizer.addAuthRule(testPath.toString(), - WasbAuthorizationOperations.EXECUTE.toString(), true); - authorizer.addAuthRule(renamePath.toString(), - WasbAuthorizationOperations.EXECUTE.toString(), true); - authorizer.addAuthRule(fs.getWorkingDirectory().toString(), - WasbAuthorizationOperations.EXECUTE.toString(), true); - fs.updateWasbAuthorizer(authorizer); - fs.create(testPath); - - Assert.check(fs.exists(testPath)); - fs.rename(testPath, renamePath); - Assert.check(fs.exists(renamePath)); - fs.delete(renamePath, false); - } - - /** - * Negative test to verify rename access check. - * @throws Throwable - */ - @Test(expected=WasbAuthorizationException.class) - public void testRenameAccessCheckNegative() throws Throwable { - - AzureBlobStorageTestAccount testAccount = createTestAccount(); - NativeAzureFileSystem fs = testAccount.getFileSystem(); - String testFile = "test.dat"; - Path testPath = new Path(fs.getWorkingDirectory(), testFile); - Path renamePath = new Path("test2.dat"); - - MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); - authorizer.init(null); - authorizer.addAuthRule(testPath.toString(), - WasbAuthorizationOperations.WRITE.toString(), true); - authorizer.addAuthRule(testPath.toString(), - WasbAuthorizationOperations.EXECUTE.toString(), false); - fs.updateWasbAuthorizer(authorizer); - - try { - fs.create(testPath); - - Assert.check(fs.exists(testPath)); - fs.rename(testPath, renamePath); - Assert.check(fs.exists(renamePath)); - fs.delete(renamePath, false); - } catch (WasbAuthorizationException ex) { - throw ex; - } finally { - authorizer = new MockWasbAuthorizerImpl(); - authorizer.init(null); - authorizer.addAuthRule(testPath.toString(), - WasbAuthorizationOperations.EXECUTE.toString(), false); - fs.updateWasbAuthorizer(authorizer); - Assert.check(fs.exists(testPath)); - fs.delete(testPath, false); - } - } - - /** - * Positive test for read access check. - * @throws Throwable - */ - @Test - public void testReadAccessCheckPositive() throws Throwable { - - AzureBlobStorageTestAccount testAccount = createTestAccount(); - NativeAzureFileSystem fs = testAccount.getFileSystem(); - String testFile = "test.dat"; - Path testPath = new Path(fs.getWorkingDirectory(), testFile); - MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); - authorizer.init(null); - authorizer.addAuthRule(testPath.toString(), - WasbAuthorizationOperations.WRITE.toString(), true); - authorizer.addAuthRule(testPath.toString(), - WasbAuthorizationOperations.EXECUTE.toString(), true); - authorizer.addAuthRule(testPath.toString(), - WasbAuthorizationOperations.READ.toString(), true); - authorizer.addAuthRule(fs.getWorkingDirectory().toString(), - WasbAuthorizationOperations.EXECUTE.toString(), true); - fs.updateWasbAuthorizer(authorizer); - fs.create(testPath); - Assert.check(fs.exists(testPath)); - FSDataInputStream inputStream = fs.open(testPath); - inputStream.close(); - fs.delete(testPath, false); - } - - /** - * Negative test to verify read access check. - * @throws Throwable - */ - @Test(expected=WasbAuthorizationException.class) - public void testReadAccessCheckNegative() throws Throwable { - - AzureBlobStorageTestAccount testAccount = createTestAccount(); - NativeAzureFileSystem fs = testAccount.getFileSystem(); - String testFile = "test.dat"; - Path testPath = new Path(fs.getWorkingDirectory(), testFile); - MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); - authorizer.init(null); - authorizer.addAuthRule(testPath.toString(), - WasbAuthorizationOperations.WRITE.toString(), true); - authorizer.addAuthRule(testPath.toString(), - WasbAuthorizationOperations.EXECUTE.toString(), true); - authorizer.addAuthRule(testPath.toString(), - WasbAuthorizationOperations.READ.toString(), false); - fs.updateWasbAuthorizer(authorizer); - - fs.create(new Path(testFile)); - Assert.check(fs.exists(testPath)); - FSDataInputStream inputStream = null; - try { - inputStream = fs.open(new Path(testFile)); - } catch (WasbAuthorizationException ex) { - throw ex; - } finally { - fs.delete(new Path(testFile), false); - if (inputStream != null) { - inputStream.close(); - } - } - } -} \ No newline at end of file From 686823529be09bea2a6cecb3503ef722017475bc Mon Sep 17 00:00:00 2001 From: Mingliang Liu Date: Mon, 6 Mar 2017 17:16:36 -0800 Subject: [PATCH 020/188] HADOOP-13930. Azure: Add Authorization support to WASB. Contributed by Sivaguru Sankaridurg and Dushyanth --- .../src/main/resources/core-default.xml | 10 + .../conf/TestCommonConfigurationFields.java | 1 + .../fs/azure/AzureNativeFileSystemStore.java | 5 +- .../fs/azure/NativeAzureFileSystem.java | 116 +++++- .../fs/azure/RemoteWasbAuthorizerImpl.java | 190 ++++++++++ .../fs/azure/WasbAuthorizationException.java | 40 ++ .../fs/azure/WasbAuthorizationOperations.java | 44 +++ .../fs/azure/WasbAuthorizerInterface.java | 53 +++ .../hadoop/fs/azure/WasbRemoteCallHelper.java | 71 +++- .../hadoop-azure/src/site/markdown/index.md | 34 ++ .../fs/azure/AzureBlobStorageTestAccount.java | 59 +-- .../fs/azure/MockWasbAuthorizerImpl.java | 102 ++++++ ...estNativeAzureFileSystemAuthorization.java | 344 ++++++++++++++++++ .../fs/azure/TestWasbRemoteCallHelper.java | 344 ++++++++++++++++++ .../src/test/resources/azure-test.xml | 28 +- 15 files changed, 1372 insertions(+), 69 deletions(-) create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationException.java create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationOperations.java create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizerInterface.java create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbRemoteCallHelper.java diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 35be56bf81e..52b58eddb1e 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -1292,6 +1292,16 @@ to specify the time (such as 2s, 2m, 1h, etc.). + + fs.azure.authorization + false + + Config flag to enable authorization support in WASB. Setting it to "true" enables + authorization support to WASB. Currently WASB authorization requires a remote service + to provide authorization that needs to be specified via fs.azure.authorization.remote.service.url + configuration + + diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java index 966a8ac0d08..cbfb6d1a7c2 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java @@ -114,6 +114,7 @@ public class TestCommonConfigurationFields extends TestConfigurationFieldsBase { xmlPropsToSkipCompare.add("fs.azure.sas.expiry.period"); xmlPropsToSkipCompare.add("fs.azure.local.sas.key.mode"); xmlPropsToSkipCompare.add("fs.azure.secure.mode"); + xmlPropsToSkipCompare.add("fs.azure.authorization"); // Deprecated properties. These should eventually be removed from the // class. diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java index 07c389ce0c2..a8708ecafdf 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java @@ -249,7 +249,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { * Default values to control SAS Key mode. * By default we set the values to false. */ - private static final boolean DEFAULT_USE_SECURE_MODE = false; + public static final boolean DEFAULT_USE_SECURE_MODE = false; private static final boolean DEFAULT_USE_LOCAL_SAS_KEY_MODE = false; /** @@ -849,6 +849,9 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { rootDirectory = container.getDirectoryReference(""); canCreateOrModifyContainer = true; + + configureAzureStorageSession(); + tolerateOobAppends = false; } /** diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java index 4184a539418..6de0a285a84 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java @@ -1106,7 +1106,31 @@ public class NativeAzureFileSystem extends FileSystem { // A counter to create unique (within-process) names for my metrics sources. private static AtomicInteger metricsSourceNameCounter = new AtomicInteger(); private boolean appendSupportEnabled = false; - + + /** + * Configuration key to enable authorization support in WASB. + */ + public static final String KEY_AZURE_AUTHORIZATION = + "fs.azure.authorization"; + + /** + * Default value for the authorization support in WASB. + */ + private static final boolean DEFAULT_AZURE_AUTHORIZATION = false; + + /** + * Flag controlling authorization support in WASB. + */ + private boolean azureAuthorization = false; + + /** + * Authorizer to use when authorization support is enabled in + * WASB. + */ + private WasbAuthorizerInterface authorizer = null; + + private String delegationToken = null; + public NativeAzureFileSystem() { // set store in initialize() } @@ -1146,11 +1170,11 @@ public class NativeAzureFileSystem extends FileSystem { return baseName + number; } } - + /** * Checks if the given URI scheme is a scheme that's affiliated with the Azure * File System. - * + * * @param scheme * The URI scheme. * @return true iff it's an Azure File System URI scheme. @@ -1167,7 +1191,7 @@ public class NativeAzureFileSystem extends FileSystem { /** * Puts in the authority of the default file system if it is a WASB file * system and the given URI's authority is null. - * + * * @return The URI with reconstructed authority if necessary and possible. */ private static URI reconstructAuthorityIfNeeded(URI uri, Configuration conf) { @@ -1237,6 +1261,24 @@ public class NativeAzureFileSystem extends FileSystem { // Initialize thread counts from user configuration deleteThreadCount = conf.getInt(AZURE_DELETE_THREADS, DEFAULT_AZURE_DELETE_THREADS); renameThreadCount = conf.getInt(AZURE_RENAME_THREADS, DEFAULT_AZURE_RENAME_THREADS); + + boolean useSecureMode = conf.getBoolean(AzureNativeFileSystemStore.KEY_USE_SECURE_MODE, + AzureNativeFileSystemStore.DEFAULT_USE_SECURE_MODE); + + this.azureAuthorization = useSecureMode && + conf.getBoolean(KEY_AZURE_AUTHORIZATION, DEFAULT_AZURE_AUTHORIZATION); + + if (this.azureAuthorization) { + + this.authorizer = + new RemoteWasbAuthorizerImpl(); + authorizer.init(conf); + } + } + + @VisibleForTesting + public void updateWasbAuthorizer(WasbAuthorizerInterface authorizer) { + this.authorizer = authorizer; } private NativeFileSystemStore createDefaultStore(Configuration conf) { @@ -1338,18 +1380,28 @@ public class NativeAzureFileSystem extends FileSystem { /** * For unit test purposes, retrieves the AzureNativeFileSystemStore store * backing this file system. - * + * * @return The store object. */ @VisibleForTesting public AzureNativeFileSystemStore getStore() { return actualStore; } - + NativeFileSystemStore getStoreInterface() { return store; } + private void performAuthCheck(String path, String accessType, + String operation) throws WasbAuthorizationException, IOException { + + if (azureAuthorization && this.authorizer != null && + !this.authorizer.authorize(path, accessType, delegationToken)) { + throw new WasbAuthorizationException(operation + + " operation for Path : " + path + " not allowed"); + } + } + /** * Gets the metrics source for this file system. * This is mainly here for unit testing purposes. @@ -1372,6 +1424,10 @@ public class NativeAzureFileSystem extends FileSystem { LOG.debug("Opening file: {} for append", f); Path absolutePath = makeAbsolute(f); + + performAuthCheck(absolutePath.toString(), + WasbAuthorizationOperations.WRITE.toString(), "append"); + String key = pathToKey(absolutePath); FileMetadata meta = null; try { @@ -1434,6 +1490,7 @@ public class NativeAzureFileSystem extends FileSystem { * Get a self-renewing lease on the specified file. * @param path path whose lease to be renewed. * @return Lease + * @throws AzureException when not being able to acquire a lease on the path */ public SelfRenewingLease acquireLease(Path path) throws AzureException { String fullKey = pathToKey(makeAbsolute(path)); @@ -1572,6 +1629,10 @@ public class NativeAzureFileSystem extends FileSystem { } Path absolutePath = makeAbsolute(f); + + performAuthCheck(absolutePath.toString(), + WasbAuthorizationOperations.WRITE.toString(), "create"); + String key = pathToKey(absolutePath); FileMetadata existingMetadata = store.retrieveMetadata(key); @@ -1652,10 +1713,10 @@ public class NativeAzureFileSystem extends FileSystem { // Construct the data output stream from the buffered output stream. FSDataOutputStream fsOut = new FSDataOutputStream(bufOutStream, statistics); - + // Increment the counter instrumentation.fileCreated(); - + // Return data output stream to caller. return fsOut; } @@ -1694,6 +1755,10 @@ public class NativeAzureFileSystem extends FileSystem { LOG.debug("Deleting file: {}", f.toString()); Path absolutePath = makeAbsolute(f); + + performAuthCheck(absolutePath.toString(), + WasbAuthorizationOperations.EXECUTE.toString(), "delete"); + String key = pathToKey(absolutePath); // Capture the metadata for the path. @@ -1964,6 +2029,10 @@ public class NativeAzureFileSystem extends FileSystem { // Capture the absolute path and the path to key. Path absolutePath = makeAbsolute(f); + + performAuthCheck(absolutePath.toString(), + WasbAuthorizationOperations.EXECUTE.toString(), "getFileStatus"); + String key = pathToKey(absolutePath); if (key.length() == 0) { // root always exists return newDirectory(null, absolutePath); @@ -2062,6 +2131,10 @@ public class NativeAzureFileSystem extends FileSystem { LOG.debug("Listing status for {}", f.toString()); Path absolutePath = makeAbsolute(f); + + performAuthCheck(absolutePath.toString(), + WasbAuthorizationOperations.EXECUTE.toString(), "list"); + String key = pathToKey(absolutePath); Set status = new TreeSet(); FileMetadata meta = null; @@ -2228,7 +2301,7 @@ public class NativeAzureFileSystem extends FileSystem { /** * Applies the applicable UMASK's on the given permission. - * + * * @param permission * The permission to mask. * @param applyMode @@ -2250,7 +2323,7 @@ public class NativeAzureFileSystem extends FileSystem { /** * Creates the PermissionStatus object to use for the given permission, based * on the current user in context. - * + * * @param permission * The permission for the file. * @return The permission status object to use. @@ -2284,6 +2357,10 @@ public class NativeAzureFileSystem extends FileSystem { } Path absolutePath = makeAbsolute(f); + + performAuthCheck(absolutePath.toString(), + WasbAuthorizationOperations.EXECUTE.toString(), "mkdirs"); + PermissionStatus permissionStatus = null; if(noUmask) { // ensure owner still has wx permissions at the minimum @@ -2337,6 +2414,10 @@ public class NativeAzureFileSystem extends FileSystem { LOG.debug("Opening file: {}", f.toString()); Path absolutePath = makeAbsolute(f); + + performAuthCheck(absolutePath.toString(), + WasbAuthorizationOperations.READ.toString(), "read"); + String key = pathToKey(absolutePath); FileMetadata meta = null; try { @@ -2393,7 +2474,12 @@ public class NativeAzureFileSystem extends FileSystem { + " through WASB that has colons in the name"); } - String srcKey = pathToKey(makeAbsolute(src)); + Path absolutePath = makeAbsolute(src); + + performAuthCheck(absolutePath.toString(), + WasbAuthorizationOperations.EXECUTE.toString(), "rename"); + + String srcKey = pathToKey(absolutePath); if (srcKey.length() == 0) { // Cannot rename root of file system @@ -2695,6 +2781,10 @@ public class NativeAzureFileSystem extends FileSystem { @Override public void setPermission(Path p, FsPermission permission) throws FileNotFoundException, IOException { Path absolutePath = makeAbsolute(p); + + performAuthCheck(absolutePath.toString(), + WasbAuthorizationOperations.EXECUTE.toString(), "setPermission"); + String key = pathToKey(absolutePath); FileMetadata metadata = null; try { @@ -2733,6 +2823,10 @@ public class NativeAzureFileSystem extends FileSystem { public void setOwner(Path p, String username, String groupname) throws IOException { Path absolutePath = makeAbsolute(p); + + performAuthCheck(absolutePath.toString(), + WasbAuthorizationOperations.EXECUTE.toString(), "setOwner"); + String key = pathToKey(absolutePath); FileMetadata metadata = null; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java new file mode 100644 index 00000000000..5f2265bc732 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java @@ -0,0 +1,190 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure; + +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.conf.Configuration; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.utils.URIBuilder; + +import java.io.IOException; +import java.net.URISyntaxException; + +import static org.apache.hadoop.fs.azure.WasbRemoteCallHelper.REMOTE_CALL_SUCCESS_CODE; + +/** + * Class implementing WasbAuthorizerInterface using a remote + * service that implements the authorization operation. This + * class expects the url of the remote service to be passed + * via config. + */ +public class RemoteWasbAuthorizerImpl implements WasbAuthorizerInterface { + + private String remoteAuthorizerServiceUrl = ""; + + /** + * Configuration parameter name expected in the Configuration object to + * provide the url of the remote service. {@value} + */ + public static final String KEY_REMOTE_AUTH_SERVICE_URL = + "fs.azure.authorization.remote.service.url"; + + /** + * Authorization operation OP name in the remote service {@value} + */ + private static final String CHECK_AUTHORIZATION_OP = + "CHECK_AUTHORIZATION"; + + /** + * Query parameter specifying the access operation type. {@value} + */ + private static final String ACCESS_OPERATION_QUERY_PARAM_NAME = + "operation_type"; + + /** + * Query parameter specifying the wasb absolute path. {@value} + */ + private static final String WASB_ABSOLUTE_PATH_QUERY_PARAM_NAME = + "wasb_absolute_path"; + + /** + * Query parameter name for user info {@value} + */ + private static final String DELEGATION_TOKEN_QUERY_PARAM_NAME = + "delegation_token"; + + private WasbRemoteCallHelper remoteCallHelper = null; + + @VisibleForTesting + public void updateWasbRemoteCallHelper(WasbRemoteCallHelper helper) { + this.remoteCallHelper = helper; + } + + @Override + public void init(Configuration conf) + throws WasbAuthorizationException, IOException { + + remoteAuthorizerServiceUrl = conf.get(KEY_REMOTE_AUTH_SERVICE_URL); + + if (remoteAuthorizerServiceUrl == null + || remoteAuthorizerServiceUrl.isEmpty()) { + throw new WasbAuthorizationException( + "fs.azure.authorization.remote.service.url config not set" + + " in configuration."); + } + + this.remoteCallHelper = new WasbRemoteCallHelper(); + } + + @Override + public boolean authorize(String wasbAbsolutePath, String accessType, + String delegationToken) throws WasbAuthorizationException, IOException { + + try { + URIBuilder uriBuilder = new URIBuilder(remoteAuthorizerServiceUrl); + uriBuilder.setPath("/" + CHECK_AUTHORIZATION_OP); + uriBuilder.addParameter(WASB_ABSOLUTE_PATH_QUERY_PARAM_NAME, + wasbAbsolutePath); + uriBuilder.addParameter(ACCESS_OPERATION_QUERY_PARAM_NAME, + accessType); + uriBuilder.addParameter(DELEGATION_TOKEN_QUERY_PARAM_NAME, + delegationToken); + + String responseBody = remoteCallHelper.makeRemoteGetRequest( + new HttpGet(uriBuilder.build())); + + ObjectMapper objectMapper = new ObjectMapper(); + RemoteAuthorizerResponse authorizerResponse = + objectMapper.readValue(responseBody, RemoteAuthorizerResponse.class); + + if (authorizerResponse == null) { + throw new WasbAuthorizationException( + "RemoteAuthorizerResponse object null from remote call"); + } else if (authorizerResponse.getResponseCode() + == REMOTE_CALL_SUCCESS_CODE) { + return authorizerResponse.getAuthorizationResult(); + } else { + throw new WasbAuthorizationException("Remote authorization" + + " service encountered an error " + + authorizerResponse.getResponseMessage()); + } + } catch (URISyntaxException | WasbRemoteCallException + | JsonParseException | JsonMappingException ex) { + throw new WasbAuthorizationException(ex); + } + } +} + +/** + * POJO representing the response expected from a remote + * authorization service. + * The remote service is expected to return the authorization + * response in the following JSON format + * { + * "responseCode" : 0 or non-zero , + * "responseMessage" : relavant message of failure + * "authorizationResult" : authorization result + * true - if auhorization allowed + * false - otherwise. + * + * } + */ +class RemoteAuthorizerResponse { + + private int responseCode; + private boolean authorizationResult; + private String responseMessage; + + public RemoteAuthorizerResponse(int responseCode, + boolean authorizationResult, String message) { + this.responseCode = responseCode; + this.authorizationResult = authorizationResult; + this.responseMessage = message; + } + + public RemoteAuthorizerResponse() { + } + + public int getResponseCode() { + return responseCode; + } + + public void setResponseCode(int responseCode) { + this.responseCode = responseCode; + } + + public boolean getAuthorizationResult() { + return authorizationResult; + } + + public void setAuthorizationResult(boolean authorizationResult) { + this.authorizationResult = authorizationResult; + } + + public String getResponseMessage() { + return responseMessage; + } + + public void setResponseMessage(String message) { + this.responseMessage = message; + } +} \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationException.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationException.java new file mode 100644 index 00000000000..eff9248dffe --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationException.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure; + +/** + * Exception that gets thrown during the authorization failures + * in WASB. + */ +public class WasbAuthorizationException extends AzureException { + + private static final long serialVersionUID = 1L; + + public WasbAuthorizationException(String message) { + super(message); + } + + public WasbAuthorizationException(String message, Throwable cause) { + super(message, cause); + } + + public WasbAuthorizationException(Throwable t) { + super(t); + } +} \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationOperations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationOperations.java new file mode 100644 index 00000000000..41ca2b32267 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizationOperations.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure; + +/** + * Different authorization operations supported + * in WASB. + */ + +public enum WasbAuthorizationOperations { + + READ, WRITE, EXECUTE; + + @Override + public String toString() { + switch(this) { + case READ: + return "read"; + case WRITE: + return "write"; + case EXECUTE: + return "execute"; + default: + throw new IllegalArgumentException( + "Invalid Authorization Operation"); + } + } +} \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizerInterface.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizerInterface.java new file mode 100644 index 00000000000..f391851095a --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizerInterface.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; + +/** + * Interface to implement authorization support in WASB. + * API's of this interface will be implemented in the + * StorageInterface Layer before making calls to Azure + * Storage. + */ +public interface WasbAuthorizerInterface { + /** + * Initializer method + * @param conf - Configuration object + * @throws WasbAuthorizationException - On authorization exceptions + * @throws IOException - When not able to reach the authorizer + */ + public void init(Configuration conf) + throws WasbAuthorizationException, IOException; + + /** + * Authorizer API to authorize access in WASB. + + * @param wasbAbolutePath : Absolute WASB Path used for access. + * @param accessType : Type of access + * @param delegationToken : The user information. + * @return : true - If access allowed false - If access is not allowed. + * @throws WasbAuthorizationException - On authorization exceptions + * @throws IOException - When not able to reach the authorizer + */ + public boolean authorize(String wasbAbolutePath, String accessType, + String delegationToken) throws WasbAuthorizationException, IOException; +} \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbRemoteCallHelper.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbRemoteCallHelper.java index 543c899f98e..09ea0847ee4 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbRemoteCallHelper.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbRemoteCallHelper.java @@ -18,18 +18,21 @@ package org.apache.hadoop.fs.azure; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; - +import com.google.common.annotations.VisibleForTesting; +import org.apache.http.Header; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; +import org.apache.http.StatusLine; import org.apache.http.client.ClientProtocolException; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.HttpClientBuilder; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + /** * Helper class the has constants and helper methods * used in WASB when integrating with a remote http cred @@ -48,6 +51,11 @@ class WasbRemoteCallHelper { */ private HttpClient client = null; + @VisibleForTesting + public void updateHttpClient(HttpClient client) { + this.client = client; + } + public WasbRemoteCallHelper() { this.client = HttpClientBuilder.create().build(); } @@ -58,17 +66,54 @@ class WasbRemoteCallHelper { * @return Http Response body returned as a string. The caller * is expected to semantically understand the response. * @throws WasbRemoteCallException + * @throws IOException */ public String makeRemoteGetRequest(HttpGet getRequest) - throws WasbRemoteCallException { + throws WasbRemoteCallException, IOException { try { + final String APPLICATION_JSON = "application/json"; + final int MAX_CONTENT_LENGTH = 1024; + + getRequest.setHeader("Accept", APPLICATION_JSON); + HttpResponse response = client.execute(getRequest); - if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { - throw new WasbRemoteCallException( - response.getStatusLine().toString()); + StatusLine statusLine = response.getStatusLine(); + if (statusLine == null || statusLine.getStatusCode() != HttpStatus.SC_OK) { + throw new WasbRemoteCallException(getRequest.getURI().toString() + ":" + + ((statusLine!=null) ? statusLine.toString() : "NULL") + ); + } + + Header contentTypeHeader = response.getFirstHeader("Content-Type"); + if (contentTypeHeader == null || contentTypeHeader.getValue() != APPLICATION_JSON) { + throw new WasbRemoteCallException(getRequest.getURI().toString() + ":" + + "Content-Type mismatch: expected: " + APPLICATION_JSON + + ", got " + ((contentTypeHeader!=null) ? contentTypeHeader.getValue() : "NULL") + ); + } + + Header contentLengthHeader = response.getFirstHeader("Content-Length"); + if (contentLengthHeader == null) { + throw new WasbRemoteCallException(getRequest.getURI().toString() + ":" + + "Content-Length header missing" + ); + } + + try { + if (Integer.parseInt(contentLengthHeader.getValue()) > MAX_CONTENT_LENGTH) { + throw new WasbRemoteCallException(getRequest.getURI().toString() + ":" + + "Content-Length:" + contentLengthHeader.getValue() + + "exceeded max:" + MAX_CONTENT_LENGTH + ); + } + } + catch (NumberFormatException nfe) { + throw new WasbRemoteCallException(getRequest.getURI().toString() + ":" + + "Invalid Content-Length value :" + contentLengthHeader.getValue() + ); } BufferedReader rd = new BufferedReader( @@ -83,11 +128,11 @@ class WasbRemoteCallHelper { return responseBody.toString(); } catch (ClientProtocolException clientProtocolEx) { - throw new WasbRemoteCallException("Encountered ClientProtocolException" - + " while making remote call", clientProtocolEx); + throw new WasbRemoteCallException(getRequest.getURI().toString() + ":" + + "Encountered ClientProtocolException while making remote call", clientProtocolEx); } catch (IOException ioEx) { - throw new WasbRemoteCallException("Encountered IOException while making" - + " remote call", ioEx); + throw new WasbRemoteCallException(getRequest.getURI().toString() + ":" + + "Encountered IOException while making remote call", ioEx); } } } \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/index.md b/hadoop-tools/hadoop-azure/src/site/markdown/index.md index 2865223acdb..1d1274b930d 100644 --- a/hadoop-tools/hadoop-azure/src/site/markdown/index.md +++ b/hadoop-tools/hadoop-azure/src/site/markdown/index.md @@ -333,6 +333,40 @@ The service is expected to return a response in JSON format: "sasKey" : Requested SAS Key } ``` + +## Authorization Support in WASB. + +Authorization support can be enabled in WASB using the following configuration: + +``` + + fs.azure.authorization + true + +``` + The current implementation of authorization relies on the presence of an external service that can enforce + the authorization. The service is expected to be running on a URL provided by the following config. + +``` + + fs.azure.authorization.remote.service.url + {URL} + +``` + + The remote service is expected to provide support for the following REST call: ```{URL}/CHECK_AUTHORIZATION``` + An example request: + ```{URL}/CHECK_AUTHORIZATION?wasb_absolute_path=&operation_type=&delegation_token=``` + + The service is expected to return a response in JSON format: + ``` + { + "responseCode" : 0 or non-zero , + "responseMessage" : relevant message on failure , + "authorizationResult" : true/false + } + ``` + ## Testing the hadoop-azure Module The hadoop-azure module includes a full suite of unit tests. Most of the tests diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java index 5353663bc89..5f66fd2f0ac 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java @@ -18,17 +18,9 @@ package org.apache.hadoop.fs.azure; -import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.DEFAULT_STORAGE_EMULATOR_ACCOUNT_NAME; - -import java.net.URI; -import java.net.URISyntaxException; -import java.util.Calendar; -import java.util.Date; -import java.util.EnumSet; -import java.util.GregorianCalendar; -import java.util.TimeZone; -import java.util.concurrent.ConcurrentLinkedQueue; - +import com.microsoft.azure.storage.*; +import com.microsoft.azure.storage.blob.*; +import com.microsoft.azure.storage.core.Base64; import org.apache.commons.configuration2.SubsetConfiguration; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -39,22 +31,14 @@ import org.apache.hadoop.metrics2.MetricsRecord; import org.apache.hadoop.metrics2.MetricsSink; import org.apache.hadoop.metrics2.MetricsTag; import org.apache.hadoop.metrics2.impl.TestMetricsConfig; -import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; -import com.microsoft.azure.storage.AccessCondition; -import com.microsoft.azure.storage.CloudStorageAccount; -import com.microsoft.azure.storage.StorageCredentials; -import com.microsoft.azure.storage.StorageCredentialsAccountAndKey; -import com.microsoft.azure.storage.StorageCredentialsAnonymous; -import com.microsoft.azure.storage.blob.BlobContainerPermissions; -import com.microsoft.azure.storage.blob.BlobContainerPublicAccessType; -import com.microsoft.azure.storage.blob.BlobOutputStream; -import com.microsoft.azure.storage.blob.CloudBlobClient; -import com.microsoft.azure.storage.blob.CloudBlobContainer; -import com.microsoft.azure.storage.blob.CloudBlockBlob; -import com.microsoft.azure.storage.blob.SharedAccessBlobPermissions; -import com.microsoft.azure.storage.blob.SharedAccessBlobPolicy; -import com.microsoft.azure.storage.core.Base64; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.*; +import java.util.concurrent.ConcurrentLinkedQueue; + +import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.DEFAULT_STORAGE_EMULATOR_ACCOUNT_NAME; +import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.KEY_USE_LOCAL_SAS_KEY_MODE; /** * Helper class to create WASB file systems backed by either a mock in-memory @@ -92,7 +76,7 @@ public final class AzureBlobStorageTestAccount { private static final ConcurrentLinkedQueue allMetrics = new ConcurrentLinkedQueue(); private static boolean metricsConfigSaved = false; - + private AzureBlobStorageTestAccount(NativeAzureFileSystem fs, CloudStorageAccount account, CloudBlobContainer container) { @@ -272,6 +256,7 @@ public final class AzureBlobStorageTestAccount { store.setAzureStorageInteractionLayer(mockStorage); NativeAzureFileSystem fs = new NativeAzureFileSystem(store); setMockAccountKey(conf); + configureSecureModeTestSettings(conf); // register the fs provider. fs.initialize(new URI(MOCK_WASB_URI), conf); @@ -332,6 +317,8 @@ public final class AzureBlobStorageTestAccount { // Set account URI and initialize Azure file system. URI accountUri = createAccountUri(DEFAULT_STORAGE_EMULATOR_ACCOUNT_NAME, containerName); + configureSecureModeTestSettings(conf); + fs.initialize(accountUri, conf); // Create test account initializing the appropriate member variables. @@ -368,6 +355,7 @@ public final class AzureBlobStorageTestAccount { // out-of-band appends. conf.setBoolean(KEY_DISABLE_THROTTLING, true); conf.setBoolean(KEY_READ_TOLERATE_CONCURRENT_APPEND, true); + configureSecureModeTestSettings(conf); // Set account URI and initialize Azure file system. URI accountUri = createAccountUri(accountName, containerName); @@ -408,6 +396,17 @@ public final class AzureBlobStorageTestAccount { setMockAccountKey(conf, MOCK_ACCOUNT_NAME); } + /** + * Configure default values for Secure Mode testing. + * These values are relevant only when testing in Secure Mode. + * + * @param conf + * The configuration. + */ + public static void configureSecureModeTestSettings(Configuration conf) { + conf.set(KEY_USE_LOCAL_SAS_KEY_MODE, "true"); // always use local sas-key mode for testing + } + /** * Sets the mock account key in the given configuration. * @@ -556,6 +555,8 @@ public final class AzureBlobStorageTestAccount { conf.setBoolean(KEY_DISABLE_THROTTLING, true); } + configureSecureModeTestSettings(conf); + // Set account URI and initialize Azure file system. URI accountUri = createAccountUri(accountName, containerName); fs.initialize(accountUri, conf); @@ -693,6 +694,8 @@ public final class AzureBlobStorageTestAccount { // Capture the account URL and the account name. String accountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME); + configureSecureModeTestSettings(conf); + // Generate a container name and create a shared access signature string for // it. // @@ -764,6 +767,8 @@ public final class AzureBlobStorageTestAccount { // Capture the account URL and the account name. String accountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME); + configureSecureModeTestSettings(conf); + // Set up public container with the specified blob name. CloudBlockBlob blobRoot = primeRootContainer(blobClient, accountName, blobName, fileSize); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java new file mode 100644 index 00000000000..8f7cb2ae5ea --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; + +/** + * A mock wasb authorizer implementation. + */ + +public class MockWasbAuthorizerImpl implements WasbAuthorizerInterface { + + private Map authRules; + + @Override + public void init(Configuration conf) { + authRules = new HashMap(); + } + + public void addAuthRule(String wasbAbsolutePath, + String accessType, boolean access) { + AuthorizationComponent component = + new AuthorizationComponent(wasbAbsolutePath, accessType); + this.authRules.put(component, access); + } + + @Override + public boolean authorize(String wasbAbsolutePath, String accessType, + String delegationToken) throws WasbAuthorizationException { + + AuthorizationComponent component = + new AuthorizationComponent(wasbAbsolutePath, accessType); + + if (authRules.containsKey(component)) { + return authRules.get(component); + } else { + return false; + } + } +} + +class AuthorizationComponent { + + private String wasbAbsolutePath; + private String accessType; + + public AuthorizationComponent(String wasbAbsolutePath, + String accessType) { + this.wasbAbsolutePath = wasbAbsolutePath; + this.accessType = accessType; + } + + @Override + public int hashCode() { + return this.wasbAbsolutePath.hashCode() ^ this.accessType.hashCode(); + } + + @Override + public boolean equals(Object obj) { + + if (obj == this) { + return true; + } + + if (obj == null + || !(obj instanceof AuthorizationComponent)) { + return false; + } + + return ((AuthorizationComponent)obj). + getWasbAbsolutePath().equals(this.wasbAbsolutePath) + && ((AuthorizationComponent)obj). + getAccessType().equals(this.accessType); + } + + public String getWasbAbsolutePath() { + return this.wasbAbsolutePath; + } + + public String getAccessType() { + return accessType; + } +} \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java new file mode 100644 index 00000000000..a2bbeb1765d --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemAuthorization.java @@ -0,0 +1,344 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.junit.Assume; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +import com.sun.tools.javac.util.Assert; +import org.junit.rules.ExpectedException; + +import java.io.Console; + +import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.KEY_USE_SECURE_MODE; + +/** + * Test class to hold all WASB authorization tests. + */ +public class TestNativeAzureFileSystemAuthorization + extends AbstractWasbTestBase { + + @Override + protected AzureBlobStorageTestAccount createTestAccount() throws Exception { + Configuration conf = new Configuration(); + conf.set(NativeAzureFileSystem.KEY_AZURE_AUTHORIZATION, "true"); + conf.set(RemoteWasbAuthorizerImpl.KEY_REMOTE_AUTH_SERVICE_URL, "http://localhost/"); + return AzureBlobStorageTestAccount.create(conf); + } + + + @Before + public void beforeMethod() { + boolean useSecureMode = fs.getConf().getBoolean(KEY_USE_SECURE_MODE, false); + boolean useAuthorization = fs.getConf().getBoolean(NativeAzureFileSystem.KEY_AZURE_AUTHORIZATION, false); + Assume.assumeTrue("Test valid when both SecureMode and Authorization are enabled .. skipping", + useSecureMode && useAuthorization); + + Assume.assumeTrue( + useSecureMode && useAuthorization + ); + } + + + @Rule + public ExpectedException expectedEx = ExpectedException.none(); + + /** + * Positive test to verify Create and delete access check + * @throws Throwable + */ + @Test + public void testCreateAccessCheckPositive() throws Throwable { + + AzureBlobStorageTestAccount testAccount = createTestAccount(); + NativeAzureFileSystem fs = testAccount.getFileSystem(); + + Path parentDir = new Path("/testCreateAccessCheckPositive"); + Path testPath = new Path(parentDir, "test.dat"); + + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + authorizer.init(null); + authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.WRITE.toString(), true); + authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true); + authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true); + fs.updateWasbAuthorizer(authorizer); + + fs.create(testPath); + ContractTestUtils.assertPathExists(fs, "testPath was not created", testPath); + fs.delete(parentDir, true); + } + + /** + * Negative test to verify Create access check + * @throws Throwable + */ + + @Test // (expected=WasbAuthorizationException.class) + public void testCreateAccessCheckNegative() throws Throwable { + + expectedEx.expect(WasbAuthorizationException.class); + expectedEx.expectMessage("create operation for Path : /testCreateAccessCheckNegative/test.dat not allowed"); + + AzureBlobStorageTestAccount testAccount = createTestAccount(); + NativeAzureFileSystem fs = testAccount.getFileSystem(); + + Path parentDir = new Path("/testCreateAccessCheckNegative"); + Path testPath = new Path(parentDir, "test.dat"); + + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + authorizer.init(null); + authorizer.addAuthRule(testPath.toString(),WasbAuthorizationOperations.WRITE.toString(), false); + authorizer.addAuthRule(parentDir.toString(),WasbAuthorizationOperations.EXECUTE.toString(), true); + fs.updateWasbAuthorizer(authorizer); + + try { + fs.create(testPath); + } + finally { + fs.delete(parentDir, true); + } + } + + /** + * Positive test to verify Create and delete access check + * @throws Throwable + */ + @Test + public void testListAccessCheckPositive() throws Throwable { + + AzureBlobStorageTestAccount testAccount = createTestAccount(); + NativeAzureFileSystem fs = testAccount.getFileSystem(); + + Path parentDir = new Path("/testListAccessCheckPositive"); + Path testPath = new Path(parentDir, "test.dat"); + + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + authorizer.init(null); + authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.WRITE.toString(), true); + authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true); + authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true); + fs.updateWasbAuthorizer(authorizer); + + fs.create(testPath); + ContractTestUtils.assertPathExists(fs, "testPath does not exist", testPath); + + try { + fs.listStatus(testPath); + } + finally { + fs.delete(parentDir, true); + } + } + + /** + * Negative test to verify Create access check + * @throws Throwable + */ + + @Test //(expected=WasbAuthorizationException.class) + public void testListAccessCheckNegative() throws Throwable { + + expectedEx.expect(WasbAuthorizationException.class); + expectedEx.expectMessage("getFileStatus operation for Path : /testListAccessCheckNegative/test.dat not allowed"); + + AzureBlobStorageTestAccount testAccount = createTestAccount(); + NativeAzureFileSystem fs = testAccount.getFileSystem(); + + Path parentDir = new Path("/testListAccessCheckNegative"); + Path testPath = new Path(parentDir, "test.dat"); + + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + authorizer.init(null); + authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.WRITE.toString(), true); + authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.EXECUTE.toString(), false); + authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true); + fs.updateWasbAuthorizer(authorizer); + + fs.create(testPath); + ContractTestUtils.assertPathExists(fs, "testPath does not exist", testPath); + + try { + fs.listStatus(testPath); + } + finally { + fs.delete(parentDir, true); + } + } + + /** + * Positive test to verify rename access check. + * @throws Throwable + */ + @Test + public void testRenameAccessCheckPositive() throws Throwable { + + AzureBlobStorageTestAccount testAccount = createTestAccount(); + NativeAzureFileSystem fs = testAccount.getFileSystem(); + + Path parentDir = new Path("/testRenameAccessCheckPositive"); + Path testPath = new Path(parentDir, "test.dat"); + Path renamePath = new Path(parentDir, "test2.dat"); + + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + authorizer.init(null); + authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.WRITE.toString(), true); + authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true); + authorizer.addAuthRule(renamePath.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true); + authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true); + fs.updateWasbAuthorizer(authorizer); + + fs.create(testPath); + ContractTestUtils.assertPathExists(fs, "sourcePath does not exist", testPath); + + try { + fs.rename(testPath, renamePath); + ContractTestUtils.assertPathExists(fs, "destPath does not exist", renamePath); + } + finally { + fs.delete(parentDir, true); + } + } + + /** + * Negative test to verify rename access check. + * @throws Throwable + */ + @Test //(expected=WasbAuthorizationException.class) + public void testRenameAccessCheckNegative() throws Throwable { + + expectedEx.expect(WasbAuthorizationException.class); + expectedEx.expectMessage("rename operation for Path : /testRenameAccessCheckNegative/test.dat not allowed"); + + AzureBlobStorageTestAccount testAccount = createTestAccount(); + NativeAzureFileSystem fs = testAccount.getFileSystem(); + Path parentDir = new Path("/testRenameAccessCheckNegative"); + Path testPath = new Path(parentDir, "test.dat"); + Path renamePath = new Path(parentDir, "test2.dat"); + + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + authorizer.init(null); + authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.WRITE.toString(), true); + // set EXECUTE to true for initial assert right after creation. + authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true); + authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true); + fs.updateWasbAuthorizer(authorizer); + + fs.create(testPath); + ContractTestUtils.assertPathExists(fs, "sourcePath does not exist", testPath); + + // Set EXECUTE to false for actual rename-failure test + authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.EXECUTE.toString(), false); + fs.updateWasbAuthorizer(authorizer); + + try { + fs.rename(testPath, renamePath); + ContractTestUtils.assertPathExists(fs, "destPath does not exist", renamePath); + } finally { + fs.delete(parentDir, true); + } + } + + /** + * Positive test for read access check. + * @throws Throwable + */ + @Test + public void testReadAccessCheckPositive() throws Throwable { + + AzureBlobStorageTestAccount testAccount = createTestAccount(); + NativeAzureFileSystem fs = testAccount.getFileSystem(); + Path parentDir = new Path("/testReadAccessCheckPositive"); + Path testPath = new Path(parentDir, "test.dat"); + + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + authorizer.init(null); + authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.WRITE.toString(), true); + authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true); + authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), true); + authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true); + fs.updateWasbAuthorizer(authorizer); + + FSDataOutputStream fso = fs.create(testPath); + String data = "Hello World"; + fso.writeBytes(data); + fso.close(); + ContractTestUtils.assertPathExists(fs, "testPath does not exist", testPath); + + FSDataInputStream inputStream = null; + try { + inputStream = fs.open(testPath); + ContractTestUtils.verifyRead(inputStream, data.getBytes(), 0, data.length()); + } + finally { + if(inputStream != null) { + inputStream.close(); + } + fs.delete(parentDir, true); + } + } + + /** + * Negative test to verify read access check. + * @throws Throwable + */ + + @Test //(expected=WasbAuthorizationException.class) + public void testReadAccessCheckNegative() throws Throwable { + + expectedEx.expect(WasbAuthorizationException.class); + expectedEx.expectMessage("read operation for Path : /testReadAccessCheckNegative/test.dat not allowed"); + + AzureBlobStorageTestAccount testAccount = createTestAccount(); + NativeAzureFileSystem fs = testAccount.getFileSystem(); + Path parentDir = new Path("/testReadAccessCheckNegative"); + Path testPath = new Path(parentDir, "test.dat"); + + MockWasbAuthorizerImpl authorizer = new MockWasbAuthorizerImpl(); + authorizer.init(null); + authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.WRITE.toString(), true); + authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true); + authorizer.addAuthRule(testPath.toString(), WasbAuthorizationOperations.READ.toString(), false); + authorizer.addAuthRule(parentDir.toString(), WasbAuthorizationOperations.EXECUTE.toString(), true); + fs.updateWasbAuthorizer(authorizer); + + FSDataOutputStream fso = fs.create(testPath); + String data = "Hello World"; + fso.writeBytes(data); + fso.close(); + ContractTestUtils.assertPathExists(fs, "testPath does not exist", testPath); + + FSDataInputStream inputStream = null; + try { + inputStream = fs.open(testPath); + ContractTestUtils.verifyRead(inputStream, data.getBytes(), 0, data.length()); + } finally { + if (inputStream != null) { + inputStream.close(); + } + fs.delete(parentDir, true); + } + } +} \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbRemoteCallHelper.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbRemoteCallHelper.java new file mode 100644 index 00000000000..b13e5e90574 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbRemoteCallHelper.java @@ -0,0 +1,344 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.apache.http.*; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.junit.Assume; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.mockito.Mockito; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; + +import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.KEY_USE_SECURE_MODE; + +/** + * Test class to hold all WasbRemoteCallHelper tests + */ +public class TestWasbRemoteCallHelper + extends AbstractWasbTestBase { + + @Override + protected AzureBlobStorageTestAccount createTestAccount() throws Exception { + Configuration conf = new Configuration(); + conf.set(NativeAzureFileSystem.KEY_AZURE_AUTHORIZATION, "true"); + conf.set(RemoteWasbAuthorizerImpl.KEY_REMOTE_AUTH_SERVICE_URL, "http://localhost/"); + return AzureBlobStorageTestAccount.create(conf); + } + + @Before + public void beforeMethod() { + boolean useSecureMode = fs.getConf().getBoolean(KEY_USE_SECURE_MODE, false); + boolean useAuthorization = fs.getConf().getBoolean(NativeAzureFileSystem.KEY_AZURE_AUTHORIZATION, false); + Assume.assumeTrue("Test valid when both SecureMode and Authorization are enabled .. skipping", + useSecureMode && useAuthorization); + + Assume.assumeTrue( + useSecureMode && useAuthorization + ); + } + + @Rule + public ExpectedException expectedEx = ExpectedException.none(); + + /** + * Test invalid status-code + * @throws Throwable + */ + @Test // (expected = WasbAuthorizationException.class) + public void testInvalidStatusCode() throws Throwable { + + setupExpectations(); + + // set up mocks + HttpClient mockHttpClient = Mockito.mock(HttpClient.class); + HttpResponse mockHttpResponse = Mockito.mock(HttpResponse.class); + Mockito.when(mockHttpClient.execute(Mockito.any())).thenReturn(mockHttpResponse); + Mockito.when(mockHttpResponse.getStatusLine()).thenReturn(newStatusLine(999)); + // finished setting up mocks + + performop(mockHttpClient); + } + + /** + * Test invalid Content-Type + * @throws Throwable + */ + @Test // (expected = WasbAuthorizationException.class) + public void testInvalidContentType() throws Throwable { + + setupExpectations(); + + // set up mocks + HttpClient mockHttpClient = Mockito.mock(HttpClient.class); + HttpResponse mockHttpResponse = Mockito.mock(HttpResponse.class); + Mockito.when(mockHttpClient.execute(Mockito.any())).thenReturn(mockHttpResponse); + Mockito.when(mockHttpResponse.getStatusLine()).thenReturn(newStatusLine(200)); + Mockito.when(mockHttpResponse.getFirstHeader("Content-Type")) + .thenReturn(newHeader("Content-Type", "text/plain")); + // finished setting up mocks + + performop(mockHttpClient); + } + + /** + * Test missing Content-Length + * @throws Throwable + */ + @Test // (expected = WasbAuthorizationException.class) + public void testMissingContentLength() throws Throwable { + + setupExpectations(); + + // set up mocks + HttpClient mockHttpClient = Mockito.mock(HttpClient.class); + HttpResponse mockHttpResponse = Mockito.mock(HttpResponse.class); + Mockito.when(mockHttpClient.execute(Mockito.any())).thenReturn(mockHttpResponse); + Mockito.when(mockHttpResponse.getStatusLine()).thenReturn(newStatusLine(200)); + Mockito.when(mockHttpResponse.getFirstHeader("Content-Type")) + .thenReturn(newHeader("Content-Type", "application/json")); + // finished setting up mocks + + performop(mockHttpClient); + } + + /** + * Test Content-Length exceeds max + * @throws Throwable + */ + @Test // (expected = WasbAuthorizationException.class) + public void testContentLengthExceedsMax() throws Throwable { + + setupExpectations(); + + // set up mocks + HttpClient mockHttpClient = Mockito.mock(HttpClient.class); + HttpResponse mockHttpResponse = Mockito.mock(HttpResponse.class); + Mockito.when(mockHttpClient.execute(Mockito.any())).thenReturn(mockHttpResponse); + Mockito.when(mockHttpResponse.getStatusLine()).thenReturn(newStatusLine(200)); + Mockito.when(mockHttpResponse.getFirstHeader("Content-Type")) + .thenReturn(newHeader("Content-Type", "application/json")); + Mockito.when(mockHttpResponse.getFirstHeader("Content-Length")) + .thenReturn(newHeader("Content-Length", "2048")); + // finished setting up mocks + + performop(mockHttpClient); + } + + /** + * Test invalid Content-Length value + * @throws Throwable + */ + @Test // (expected = WasbAuthorizationException.class) + public void testInvalidContentLengthValue() throws Throwable { + + setupExpectations(); + + // set up mocks + HttpClient mockHttpClient = Mockito.mock(HttpClient.class); + HttpResponse mockHttpResponse = Mockito.mock(HttpResponse.class); + Mockito.when(mockHttpClient.execute(Mockito.any())).thenReturn(mockHttpResponse); + Mockito.when(mockHttpResponse.getStatusLine()).thenReturn(newStatusLine(200)); + Mockito.when(mockHttpResponse.getFirstHeader("Content-Type")) + .thenReturn(newHeader("Content-Type", "application/json")); + Mockito.when(mockHttpResponse.getFirstHeader("Content-Length")) + .thenReturn(newHeader("Content-Length", "20abc48")); + // finished setting up mocks + + performop(mockHttpClient); + } + + /** + * Test valid JSON response + * @throws Throwable + */ + @Test + public void testValidJSONResponse() throws Throwable { + + // set up mocks + HttpClient mockHttpClient = Mockito.mock(HttpClient.class); + + HttpResponse mockHttpResponse = Mockito.mock(HttpResponse.class); + HttpEntity mockHttpEntity = Mockito.mock(HttpEntity.class); + + Mockito.when(mockHttpClient.execute(Mockito.any())).thenReturn(mockHttpResponse); + Mockito.when(mockHttpResponse.getStatusLine()).thenReturn(newStatusLine(200)); + Mockito.when(mockHttpResponse.getFirstHeader("Content-Type")) + .thenReturn(newHeader("Content-Type", "application/json")); + Mockito.when(mockHttpResponse.getFirstHeader("Content-Length")) + .thenReturn(newHeader("Content-Length", "1024")); + Mockito.when(mockHttpResponse.getEntity()).thenReturn(mockHttpEntity); + Mockito.when(mockHttpEntity.getContent()) + .thenReturn(new ByteArrayInputStream(validJsonResponse().getBytes(StandardCharsets.UTF_8))) + .thenReturn(new ByteArrayInputStream(validJsonResponse().getBytes(StandardCharsets.UTF_8))) + .thenReturn(new ByteArrayInputStream(validJsonResponse().getBytes(StandardCharsets.UTF_8))); + // finished setting up mocks + + performop(mockHttpClient); + } + + /** + * Test malformed JSON response + * @throws Throwable + */ + @Test // (expected = WasbAuthorizationException.class) + public void testMalFormedJSONResponse() throws Throwable { + + expectedEx.expect(WasbAuthorizationException.class); + expectedEx.expectMessage("com.fasterxml.jackson.core.JsonParseException: Unexpected end-of-input in FIELD_NAME"); + + // set up mocks + HttpClient mockHttpClient = Mockito.mock(HttpClient.class); + + HttpResponse mockHttpResponse = Mockito.mock(HttpResponse.class); + HttpEntity mockHttpEntity = Mockito.mock(HttpEntity.class); + + Mockito.when(mockHttpClient.execute(Mockito.any())).thenReturn(mockHttpResponse); + Mockito.when(mockHttpResponse.getStatusLine()).thenReturn(newStatusLine(200)); + Mockito.when(mockHttpResponse.getFirstHeader("Content-Type")) + .thenReturn(newHeader("Content-Type", "application/json")); + Mockito.when(mockHttpResponse.getFirstHeader("Content-Length")) + .thenReturn(newHeader("Content-Length", "1024")); + Mockito.when(mockHttpResponse.getEntity()).thenReturn(mockHttpEntity); + Mockito.when(mockHttpEntity.getContent()) + .thenReturn(new ByteArrayInputStream(malformedJsonResponse().getBytes(StandardCharsets.UTF_8))); + // finished setting up mocks + + performop(mockHttpClient); + } + + /** + * Test valid JSON response failure response code + * @throws Throwable + */ + @Test // (expected = WasbAuthorizationException.class) + public void testFailureCodeJSONResponse() throws Throwable { + + expectedEx.expect(WasbAuthorizationException.class); + expectedEx.expectMessage("Remote authorization service encountered an error Unauthorized"); + + // set up mocks + HttpClient mockHttpClient = Mockito.mock(HttpClient.class); + + HttpResponse mockHttpResponse = Mockito.mock(HttpResponse.class); + HttpEntity mockHttpEntity = Mockito.mock(HttpEntity.class); + + Mockito.when(mockHttpClient.execute(Mockito.any())).thenReturn(mockHttpResponse); + Mockito.when(mockHttpResponse.getStatusLine()).thenReturn(newStatusLine(200)); + Mockito.when(mockHttpResponse.getFirstHeader("Content-Type")) + .thenReturn(newHeader("Content-Type", "application/json")); + Mockito.when(mockHttpResponse.getFirstHeader("Content-Length")) + .thenReturn(newHeader("Content-Length", "1024")); + Mockito.when(mockHttpResponse.getEntity()).thenReturn(mockHttpEntity); + Mockito.when(mockHttpEntity.getContent()) + .thenReturn(new ByteArrayInputStream(failureCodeJsonResponse().getBytes(StandardCharsets.UTF_8))); + // finished setting up mocks + + performop(mockHttpClient); + } + + private void setupExpectations() { + expectedEx.expect(WasbAuthorizationException.class); + expectedEx.expectMessage("org.apache.hadoop.fs.azure.WasbRemoteCallException: " + + "http://localhost/CHECK_AUTHORIZATION?wasb_absolute_path=%2Ftest.dat&" + + "operation_type=write&delegation_token:Encountered IOException while making remote call"); + } + + private void performop(HttpClient mockHttpClient) throws Throwable { + AzureBlobStorageTestAccount testAccount = createTestAccount(); + NativeAzureFileSystem fs = testAccount.getFileSystem(); + + Path testPath = new Path("/", "test.dat"); + + RemoteWasbAuthorizerImpl authorizer = new RemoteWasbAuthorizerImpl(); + authorizer.init(fs.getConf()); + WasbRemoteCallHelper mockWasbRemoteCallHelper = new WasbRemoteCallHelper(); + mockWasbRemoteCallHelper.updateHttpClient(mockHttpClient); + authorizer.updateWasbRemoteCallHelper(mockWasbRemoteCallHelper); + fs.updateWasbAuthorizer(authorizer); + + fs.create(testPath); + ContractTestUtils.assertPathExists(fs, "testPath was not created", testPath); + fs.delete(testPath, false); + } + + private String validJsonResponse() { + return new String( + "{\"responseCode\": 0, \"authorizationResult\": true, \"responseMessage\": \"Authorized\"}" + ); + } + + private String malformedJsonResponse() { + return new String( + "{\"responseCode\": 0, \"authorizationResult\": true, \"responseMessage\":" + ); + } + + private String failureCodeJsonResponse() { + return new String( + "{\"responseCode\": 1, \"authorizationResult\": false, \"responseMessage\": \"Unauthorized\"}" + ); + } + + private StatusLine newStatusLine(int statusCode) { + return new StatusLine() { + @Override + public ProtocolVersion getProtocolVersion() { + return new ProtocolVersion("HTTP", 1, 1); + } + + @Override + public int getStatusCode() { + return statusCode; + } + + @Override + public String getReasonPhrase() { + return "Reason Phrase"; + } + }; + } + + private Header newHeader(String name, String value) { + return new Header() { + @Override + public String getName() { + return name; + } + + @Override + public String getValue() { + return value; + } + + @Override + public HeaderElement[] getElements() throws ParseException { + return new HeaderElement[0]; + } + }; + } +} \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml b/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml index e898aa631e5..c73d6d84891 100644 --- a/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml +++ b/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml @@ -19,27 +19,21 @@ + fs.azure.secure.mode false - - fs.azure.local.sas.key.mode - false - - - fs.azure.cred.service.url - {CRED_SERIVCE_URL} - - --> + From 14413989cac9acc1fa6f8d330fac32f772613325 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Tue, 7 Mar 2017 13:10:59 +0900 Subject: [PATCH 021/188] MAPREDUCE-6855. Specify charset when create String in CredentialsTestJob. Contributed by Kai Sasaki. --- .../apache/hadoop/mapreduce/security/CredentialsTestJob.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/CredentialsTestJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/CredentialsTestJob.java index e66fb2fbf3e..755e2dfba65 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/CredentialsTestJob.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/CredentialsTestJob.java @@ -19,6 +19,7 @@ package org.apache.hadoop.mapreduce.security; import java.io.IOException; +import java.nio.charset.StandardCharsets; import org.apache.hadoop.conf.Configuration; @@ -64,7 +65,7 @@ public class CredentialsTestJob extends Configured implements Tool { // fail the test } - String secretValueStr = new String (secretValue); + String secretValueStr = new String (secretValue, StandardCharsets.UTF_8); System.out.println(secretValueStr); if ( !("password"+i).equals(secretValueStr)){ From f01a69f84f4cc7d925d078a7ce32e5800da4e429 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Tue, 7 Mar 2017 13:22:11 +0900 Subject: [PATCH 022/188] Treat encrypted files as private. Contributed by Daniel Templeton. --- .../ClientDistributedCacheManager.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/ClientDistributedCacheManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/ClientDistributedCacheManager.java index 73a0330396d..9f8edb5df0d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/ClientDistributedCacheManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/ClientDistributedCacheManager.java @@ -294,10 +294,21 @@ public class ClientDistributedCacheManager { FsAction action, Map statCache) throws IOException { FileStatus status = getFileStatus(fs, path.toUri(), statCache); FsPermission perms = status.getPermission(); - FsAction otherAction = perms.getOtherAction(); - if (otherAction.implies(action)) { - return true; + + // Encrypted files are always treated as private. This stance has two + // important side effects. The first is that the encrypted files will be + // downloaded as the job owner instead of the YARN user, which is required + // for the KMS ACLs to work as expected. Second, it prevent a file with + // world readable permissions that is stored in an encryption zone from + // being localized as a publicly shared file with world readable + // permissions. + if (!perms.getEncryptedBit()) { + FsAction otherAction = perms.getOtherAction(); + if (otherAction.implies(action)) { + return true; + } } + return false; } From f597f4c43e0a6e2304b9bcaf727d6d8d15a365f9 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Tue, 7 Mar 2017 15:14:55 +0900 Subject: [PATCH 023/188] HADOOP-14087. S3A typo in pom.xml test exclusions. Contributed by Aaron Fabbri. --- hadoop-tools/hadoop-aws/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml index e5bbbfdffb9..c1880552b12 100644 --- a/hadoop-tools/hadoop-aws/pom.xml +++ b/hadoop-tools/hadoop-aws/pom.xml @@ -184,7 +184,7 @@ **/ITest*Root*.java **/ITestS3AFileContextStatistics.java **/ITestS3AEncryptionSSE*.java - **/ITestS3AHuge*.java + **/ITestS3AHuge*.java From 959940b0ab563b4e42bace44f1dc9a8babcaa889 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Tue, 7 Mar 2017 10:12:35 -0800 Subject: [PATCH 024/188] HDFS-11477. Simplify file IO profiling configuration. Contributed by Hanisha Koneru. --- .../hadoop-common/src/site/markdown/Metrics.md | 7 ++++++- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 6 +----- .../apache/hadoop/hdfs/server/common/Util.java | 17 +++++++++++++++++ .../hadoop/hdfs/server/datanode/DNConf.java | 7 ++++--- .../hdfs/server/datanode/FileIoProvider.java | 2 +- .../server/datanode/ProfilingFileIoEvents.java | 11 +++++------ .../datanode/TestDataNodeVolumeMetrics.java | 4 ++-- .../hadoop/tools/TestHdfsConfigFields.java | 2 -- 8 files changed, 36 insertions(+), 20 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md index 7900692b9f5..a8bdbebdc7d 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md @@ -332,7 +332,12 @@ Each metrics record contains tags such as SessionId and Hostname as additional i FsVolume -------- -Per-volume metrics contain Datanode Volume IO related statistics. Per-volume metrics are off by default. They can be enbabled by setting `dfs.datanode.enable.fileio.profiling` to **true**, but enabling per-volume metrics may have a performance impact. Each metrics record contains tags such as Hostname as additional information along with metrics. +Per-volume metrics contain Datanode Volume IO related statistics. Per-volume +metrics are off by default. They can be enabled by setting `dfs.datanode +.fileio.profiling.sampling.fraction` to a fraction between 0.0 and 1.0. +Setting this value to 0.0 would mean profiling is not enabled. But enabling +per-volume metrics may have a performance impact. Each metrics record +contains tags such as Hostname as additional information along with metrics. | Name | Description | |:---- |:---- | diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index be20829b6a9..82d6073399a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -718,10 +718,6 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final long DFS_EDIT_LOG_TRANSFER_RATE_DEFAULT = 0; //no throttling // Datanode File IO Stats - public static final String DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY = - "dfs.datanode.enable.fileio.profiling"; - public static final boolean DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT = - false; public static final String DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY = "dfs.datanode.enable.fileio.fault.injection"; public static final boolean @@ -730,7 +726,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys { DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY = "dfs.datanode.fileio.profiling.sampling.fraction"; public static final double - DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEAFULT = 1.0; + DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEFAULT = 0.0; //Keys with no defaults public static final String DFS_DATANODE_PLUGINS_KEY = "dfs.datanode.plugins"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java index 9c67f0a2ed7..fdb09df9dae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java @@ -388,4 +388,21 @@ public final class Util { addrsList.removeAll(addrsToExclude); return addrsList; } + + public static boolean isDiskStatsEnabled(double fileIOSamplingFraction) { + final boolean isEnabled; + if (fileIOSamplingFraction < 0.000001) { + LOG.info(DFSConfigKeys + .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY + " set to " + + fileIOSamplingFraction + ". Disabling file IO profiling"); + isEnabled = false; + } else { + LOG.info(DFSConfigKeys + .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY + " set to " + + fileIOSamplingFraction + ". Enabling file IO profiling"); + isEnabled = true; + } + + return isEnabled; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java index 3275ba8f77c..21ffccc2135 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java @@ -63,6 +63,7 @@ import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.datatransfer.TrustedChannelResolver; import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.DataTransferSaslUtil; +import org.apache.hadoop.hdfs.server.common.Util; import org.apache.hadoop.security.SaslPropertiesResolver; import java.util.concurrent.TimeUnit; @@ -174,9 +175,9 @@ public class DNConf { this.peerStatsEnabled = getConf().getBoolean( DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY, DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_DEFAULT); - this.diskStatsEnabled = getConf().getBoolean( - DFSConfigKeys.DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY, - DFSConfigKeys.DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT); + this.diskStatsEnabled = Util.isDiskStatsEnabled(getConf().getDouble( + DFSConfigKeys.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY, + DFSConfigKeys.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEFAULT)); this.outliersReportIntervalMs = getConf().getTimeDuration( DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY, DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_DEFAULT, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java index 9def2e131f7..5508e0bdd2c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FileIoProvider.java @@ -62,7 +62,7 @@ import static org.apache.hadoop.hdfs.server.datanode.FileIoProvider.OPERATION.*; * * Behavior can be injected into these events by enabling the * profiling and/or fault injection event hooks through - * {@link DFSConfigKeys#DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY} and + * {@link DFSConfigKeys#DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY} and * {@link DFSConfigKeys#DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY}. * These event hooks are disabled by default. * diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java index 9af84fa17fe..35118b2171c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ProfilingFileIoEvents.java @@ -23,6 +23,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.server.common.Util; import org.apache.hadoop.hdfs.server.datanode.fsdataset.DataNodeVolumeMetrics; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.util.Time; @@ -43,13 +44,11 @@ class ProfilingFileIoEvents { public ProfilingFileIoEvents(@Nullable Configuration conf) { if (conf != null) { - isEnabled = conf.getBoolean(DFSConfigKeys - .DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY, DFSConfigKeys - .DFS_DATANODE_ENABLE_FILEIO_PROFILING_DEFAULT); - double fileIOSamplingFraction = conf.getDouble(DFSConfigKeys - .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY, + double fileIOSamplingFraction = conf.getDouble( + DFSConfigKeys.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY, DFSConfigKeys - .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEAFULT); + .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEFAULT); + isEnabled = Util.isDiskStatsEnabled(fileIOSamplingFraction); if (fileIOSamplingFraction > 1) { LOG.warn(DFSConfigKeys .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java index 6a8ac9c4f59..03e1deea96f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeMetrics.java @@ -121,8 +121,8 @@ public class TestDataNodeVolumeMetrics { private MiniDFSCluster setupClusterForVolumeMetrics() throws IOException { Configuration conf = new HdfsConfiguration(); - conf.setBoolean(DFSConfigKeys - .DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY, true); + conf.setDouble(DFSConfigKeys + .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY, 1.0); SimulatedFSDataset.setFactory(conf); return new MiniDFSCluster.Builder(conf) .numDataNodes(NUM_DATANODES) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java index cdce3428e90..1fdf713562b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java @@ -103,8 +103,6 @@ public class TestHdfsConfigFields extends TestConfigurationFieldsBase { .add(DFSConfigKeys.DFS_DATANODE_STARTUP_KEY); configurationPropsToSkipCompare .add(DFSConfigKeys.DFS_NAMENODE_STARTUP_KEY); - configurationPropsToSkipCompare - .add(DFSConfigKeys.DFS_DATANODE_ENABLE_FILEIO_PROFILING_KEY); configurationPropsToSkipCompare.add(DFSConfigKeys .DFS_DATANODE_ENABLE_FILEIO_FAULT_INJECTION_KEY); configurationPropsToSkipCompare.add(DFSConfigKeys From 1f9848dfe1fc9148cbbcfcc3dfed948b9e0f3c3c Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Tue, 7 Mar 2017 11:41:05 -0800 Subject: [PATCH 025/188] HDFS-11508. Fix bind failure in SimpleTCPServer & Portmap where bind fails because socket is in TIME_WAIT state. Contributed by Mukul Kumar Singh. --- .../src/main/java/org/apache/hadoop/oncrpc/SimpleTcpServer.java | 1 + .../src/main/java/org/apache/hadoop/portmap/Portmap.java | 2 ++ 2 files changed, 3 insertions(+) diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpServer.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpServer.java index 99d1d6f0830..f7ab52e3d50 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpServer.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpServer.java @@ -81,6 +81,7 @@ public class SimpleTcpServer { }); server.setOption("child.tcpNoDelay", true); server.setOption("child.keepAlive", true); + server.setOption("reuseAddress", true); // Listen to TCP port ch = server.bind(new InetSocketAddress(port)); diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/Portmap.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/Portmap.java index 2b887914f99..94d76d08fdd 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/Portmap.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/Portmap.java @@ -109,12 +109,14 @@ final class Portmap { RpcUtil.STAGE_RPC_TCP_RESPONSE); } }); + tcpServer.setOption("reuseAddress", true); udpServer = new ConnectionlessBootstrap(new NioDatagramChannelFactory( Executors.newCachedThreadPool())); udpServer.setPipeline(Channels.pipeline(RpcUtil.STAGE_RPC_MESSAGE_PARSER, handler, RpcUtil.STAGE_RPC_UDP_RESPONSE)); + udpServer.setOption("reuseAddress", true); tcpChannel = tcpServer.bind(tcpAddress); udpChannel = udpServer.bind(udpAddress); From e0c239cdbda336e09a35d112d451c2e17d74a3fc Mon Sep 17 00:00:00 2001 From: Daniel Templeton Date: Tue, 7 Mar 2017 11:58:48 -0800 Subject: [PATCH 026/188] YARN-6287. RMCriticalThreadUncaughtExceptionHandler.rmContext should be final (Contributed by Corey Barker via Daniel Templeton) --- .../RMCriticalThreadUncaughtExceptionHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMCriticalThreadUncaughtExceptionHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMCriticalThreadUncaughtExceptionHandler.java index c5c60876a30..a67f81a8f7f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMCriticalThreadUncaughtExceptionHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMCriticalThreadUncaughtExceptionHandler.java @@ -37,7 +37,7 @@ public class RMCriticalThreadUncaughtExceptionHandler implements UncaughtExceptionHandler { private static final Log LOG = LogFactory.getLog( RMCriticalThreadUncaughtExceptionHandler.class); - private RMContext rmContext; + private final RMContext rmContext; public RMCriticalThreadUncaughtExceptionHandler(RMContext rmContext) { this.rmContext = rmContext; From 38d75dfd3a643f8a1acd52e025a466d65065b60e Mon Sep 17 00:00:00 2001 From: Robert Kanter Date: Tue, 7 Mar 2017 13:34:46 -0800 Subject: [PATCH 027/188] MAPREDUCE-6839. TestRecovery.testCrashed failed (pairg via rkanter) --- .../apache/hadoop/mapreduce/v2/app/TestRecovery.java | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java index 071575a156a..6332c5d825e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRecovery.java @@ -159,9 +159,7 @@ public class TestRecovery { app.waitForState(task1Attempt1, TaskAttemptState.RUNNING); app.waitForState(task2Attempt, TaskAttemptState.RUNNING); - // reduces must be in NEW state - Assert.assertEquals("Reduce Task state not correct", - TaskState.RUNNING, reduceTask.getReport().getTaskState()); + app.waitForState(reduceTask, TaskState.RUNNING); /////////// Play some games with the TaskAttempts of the first task ////// //send the fail signal to the 1st map task attempt @@ -1301,9 +1299,7 @@ public class TestRecovery { app.waitForState(task1Attempt2, TaskAttemptState.RUNNING); app.waitForState(task2Attempt, TaskAttemptState.RUNNING); - // reduces must be in NEW state - Assert.assertEquals("Reduce Task state not correct", - TaskState.RUNNING, reduceTask.getReport().getTaskState()); + app.waitForState(reduceTask, TaskState.RUNNING); //send the done signal to the map 1 attempt 1 app.getContext().getEventHandler().handle( @@ -1431,9 +1427,7 @@ public class TestRecovery { app.waitForState(task1Attempt, TaskAttemptState.RUNNING); app.waitForState(task2Attempt, TaskAttemptState.RUNNING); - // reduces must be in NEW state - Assert.assertEquals("Reduce Task state not correct", - TaskState.RUNNING, reduceTask.getReport().getTaskState()); + app.waitForState(reduceTask, TaskState.RUNNING); //send the done signal to the 1st map attempt app.getContext().getEventHandler().handle( From 1598fd3b7948b3592775e3be3227c4a336122bc9 Mon Sep 17 00:00:00 2001 From: Robert Kanter Date: Tue, 7 Mar 2017 13:47:52 -0800 Subject: [PATCH 028/188] YARN-6275. Fail to show real-time tracking charts in SLS (yufeigu via rkanter) --- hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh | 8 ++++++-- .../java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java | 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh b/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh index 19b5c345321..fb53045ed37 100644 --- a/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh +++ b/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh @@ -103,12 +103,16 @@ function run_simulation() { hadoop_java_exec sls org.apache.hadoop.yarn.sls.SLSRunner ${args} } +this="${BASH_SOURCE-$0}" +bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) + +# copy 'html' directory to current directory to make sure web sever can access +cp -r "${bin}/../html" "$(pwd)" + # let's locate libexec... if [[ -n "${HADOOP_HOME}" ]]; then HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec" else - this="${BASH_SOURCE-$0}" - bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../../../../../libexec" fi diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java index abdf1060afc..33d48466c20 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java @@ -39,6 +39,7 @@ import org.apache.hadoop.yarn.sls.SLSRunner; import org.apache.hadoop.yarn.sls.scheduler.FairSchedulerMetrics; import org.apache.hadoop.yarn.sls.scheduler.SchedulerMetrics; import org.apache.hadoop.yarn.sls.scheduler.SchedulerWrapper; +import org.eclipse.jetty.http.MimeTypes; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.Server; @@ -118,6 +119,7 @@ public class SLSWebApp extends HttpServlet { public void start() throws Exception { // static files final ResourceHandler staticHandler = new ResourceHandler(); + staticHandler.setMimeTypes(new MimeTypes()); staticHandler.setResourceBase("html"); Handler handler = new AbstractHandler() { From 28daaf0eb206d723d2baf0f9d91e43d98bb2fd26 Mon Sep 17 00:00:00 2001 From: Mingliang Liu Date: Tue, 7 Mar 2017 14:55:52 -0800 Subject: [PATCH 029/188] HADOOP-14150. Implement getHomeDirectory() method in NativeAzureFileSystem. Contributed by Santhosh G Nayak --- .../apache/hadoop/fs/azure/NativeAzureFileSystem.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java index 6de0a285a84..9aebbb5a22e 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java @@ -84,6 +84,7 @@ import com.microsoft.azure.storage.StorageException; @InterfaceStability.Stable public class NativeAzureFileSystem extends FileSystem { private static final int USER_WX_PERMISION = 0300; + private static final String USER_HOME_DIR_PREFIX_DEFAULT = "/user"; /** * A description of a folder rename operation, including the source and * destination keys, and descriptions of the files in the source folder. @@ -1129,6 +1130,8 @@ public class NativeAzureFileSystem extends FileSystem { */ private WasbAuthorizerInterface authorizer = null; + private UserGroupInformation ugi; + private String delegationToken = null; public NativeAzureFileSystem() { @@ -1247,6 +1250,7 @@ public class NativeAzureFileSystem extends FileSystem { store.initialize(uri, conf, instrumentation); setConf(conf); + this.ugi = UserGroupInformation.getCurrentUser(); this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority()); this.workingDir = new Path("/user", UserGroupInformation.getCurrentUser() .getShortUserName()).makeQualified(getUri(), getWorkingDirectory()); @@ -1276,6 +1280,12 @@ public class NativeAzureFileSystem extends FileSystem { } } + @Override + public Path getHomeDirectory() { + return makeQualified(new Path( + USER_HOME_DIR_PREFIX_DEFAULT + "/" + this.ugi.getShortUserName())); + } + @VisibleForTesting public void updateWasbAuthorizer(WasbAuthorizerInterface authorizer) { this.authorizer = authorizer; From 1eb81867032b016a59662043cbae50daa52dafa9 Mon Sep 17 00:00:00 2001 From: Sunil G Date: Wed, 8 Mar 2017 12:04:30 +0530 Subject: [PATCH 030/188] YARN-6207. Move application across queues should handle delayed event processing. Contributed by Bibin A Chundatt. --- .../SchedulerApplicationAttempt.java | 5 +- .../scheduler/capacity/CapacityScheduler.java | 69 +++--- .../capacity/TestCapacityScheduler.java | 200 ++++++++++++++++++ 3 files changed, 248 insertions(+), 26 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index f894a40f09b..91e29d59ab7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -1069,6 +1069,7 @@ public class SchedulerApplicationAttempt implements SchedulableEntity { QueueMetrics newMetrics = newQueue.getMetrics(); String newQueueName = newQueue.getQueueName(); String user = getUser(); + for (RMContainer liveContainer : liveContainers.values()) { Resource resource = liveContainer.getContainer().getResource(); ((RMContainerImpl) liveContainer).setQueueName(newQueueName); @@ -1084,7 +1085,9 @@ public class SchedulerApplicationAttempt implements SchedulableEntity { } } - appSchedulingInfo.move(newQueue); + if (!isStopped) { + appSchedulingInfo.move(newQueue); + } this.queue = newQueue; } finally { writeLock.unlock(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 20ea607cd4d..f6e79426865 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -1939,36 +1939,47 @@ public class CapacityScheduler extends String targetQueueName) throws YarnException { try { writeLock.lock(); - FiCaSchedulerApp app = getApplicationAttempt( - ApplicationAttemptId.newInstance(appId, 0)); - String sourceQueueName = app.getQueue().getQueueName(); - LeafQueue source = this.queueManager.getAndCheckLeafQueue( - sourceQueueName); + SchedulerApplication application = + applications.get(appId); + if (application == null) { + throw new YarnException("App to be moved " + appId + " not found."); + } + String sourceQueueName = application.getQueue().getQueueName(); + LeafQueue source = + this.queueManager.getAndCheckLeafQueue(sourceQueueName); String destQueueName = handleMoveToPlanQueue(targetQueueName); LeafQueue dest = this.queueManager.getAndCheckLeafQueue(destQueueName); - String user = app.getUser(); + String user = application.getUser(); try { dest.submitApplication(appId, user, destQueueName); } catch (AccessControlException e) { throw new YarnException(e); } - // Move all live containers - for (RMContainer rmContainer : app.getLiveContainers()) { - source.detachContainer(getClusterResource(), app, rmContainer); - // attach the Container to another queue - dest.attachContainer(getClusterResource(), app, rmContainer); + + FiCaSchedulerApp app = application.getCurrentAppAttempt(); + if (app != null) { + // Move all live containers even when stopped. + // For transferStateFromPreviousAttempt required + for (RMContainer rmContainer : app.getLiveContainers()) { + source.detachContainer(getClusterResource(), app, rmContainer); + // attach the Container to another queue + dest.attachContainer(getClusterResource(), app, rmContainer); + } + if (!app.isStopped()) { + source.finishApplicationAttempt(app, sourceQueueName); + // Submit to a new queue + dest.submitApplicationAttempt(app, user); + } + // Finish app & update metrics + app.move(dest); } + source.appFinished(); // Detach the application.. - source.finishApplicationAttempt(app, sourceQueueName); - source.getParent().finishApplication(appId, app.getUser()); - // Finish app & update metrics - app.move(dest); - // Submit to a new queue - dest.submitApplicationAttempt(app, user); - applications.get(appId).setQueue(dest); - LOG.info("App: " + app.getApplicationId() + " successfully moved from " - + sourceQueueName + " to: " + destQueueName); + source.getParent().finishApplication(appId, user); + application.setQueue(dest); + LOG.info("App: " + appId + " successfully moved from " + sourceQueueName + + " to: " + destQueueName); return targetQueueName; } finally { writeLock.unlock(); @@ -1980,15 +1991,23 @@ public class CapacityScheduler extends String newQueue) throws YarnException { try { writeLock.lock(); - FiCaSchedulerApp app = getApplicationAttempt( - ApplicationAttemptId.newInstance(appId, 0)); - String sourceQueueName = app.getQueue().getQueueName(); + SchedulerApplication application = + applications.get(appId); + if (application == null) { + throw new YarnException("App to be moved " + appId + " not found."); + } + String sourceQueueName = application.getQueue().getQueueName(); this.queueManager.getAndCheckLeafQueue(sourceQueueName); String destQueueName = handleMoveToPlanQueue(newQueue); LeafQueue dest = this.queueManager.getAndCheckLeafQueue(destQueueName); // Validation check - ACLs, submission limits for user & queue - String user = app.getUser(); - checkQueuePartition(app, dest); + String user = application.getUser(); + // Check active partition only when attempt is available + FiCaSchedulerApp appAttempt = + getApplicationAttempt(ApplicationAttemptId.newInstance(appId, 0)); + if (null != appAttempt) { + checkQueuePartition(appAttempt, dest); + } try { dest.validateSubmitApplication(appId, user, destQueueName); } catch (AccessControlException e) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index 2b60ecfa6c3..293bac21174 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.ContainerUpdateType; +import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Priority; @@ -110,12 +111,14 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptS import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdateEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; @@ -2209,6 +2212,203 @@ public class TestCapacityScheduler { rm.stop(); } + @Test(timeout = 60000) + public void testMoveAttemptNotAdded() throws Exception { + Configuration conf = new Configuration(); + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + MockRM rm = new MockRM(getCapacityConfiguration(conf)); + rm.start(); + CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); + + ApplicationId appId = BuilderUtils.newApplicationId(100, 1); + ApplicationAttemptId appAttemptId = + BuilderUtils.newApplicationAttemptId(appId, 1); + + RMAppAttemptMetrics attemptMetric = + new RMAppAttemptMetrics(appAttemptId, rm.getRMContext()); + RMAppImpl app = mock(RMAppImpl.class); + when(app.getApplicationId()).thenReturn(appId); + RMAppAttemptImpl attempt = mock(RMAppAttemptImpl.class); + Container container = mock(Container.class); + when(attempt.getMasterContainer()).thenReturn(container); + ApplicationSubmissionContext submissionContext = + mock(ApplicationSubmissionContext.class); + when(attempt.getSubmissionContext()).thenReturn(submissionContext); + when(attempt.getAppAttemptId()).thenReturn(appAttemptId); + when(attempt.getRMAppAttemptMetrics()).thenReturn(attemptMetric); + when(app.getCurrentAppAttempt()).thenReturn(attempt); + + rm.getRMContext().getRMApps().put(appId, app); + + SchedulerEvent addAppEvent = + new AppAddedSchedulerEvent(appId, "a1", "user"); + try { + cs.moveApplication(appId, "b1"); + fail("Move should throw exception app not available"); + } catch (YarnException e) { + assertEquals("App to be moved application_100_0001 not found.", + e.getMessage()); + } + cs.handle(addAppEvent); + cs.moveApplication(appId, "b1"); + SchedulerEvent addAttemptEvent = + new AppAttemptAddedSchedulerEvent(appAttemptId, false); + cs.handle(addAttemptEvent); + CSQueue rootQ = cs.getRootQueue(); + CSQueue queueB = cs.getQueue("b"); + CSQueue queueA = cs.getQueue("a"); + CSQueue queueA1 = cs.getQueue("a1"); + CSQueue queueB1 = cs.getQueue("b1"); + Assert.assertEquals(1, rootQ.getNumApplications()); + Assert.assertEquals(0, queueA.getNumApplications()); + Assert.assertEquals(1, queueB.getNumApplications()); + Assert.assertEquals(0, queueA1.getNumApplications()); + Assert.assertEquals(1, queueB1.getNumApplications()); + + rm.close(); + } + + @Test + public void testRemoveAttemptMoveAdded() throws Exception { + YarnConfiguration conf = new YarnConfiguration(); + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + CapacityScheduler.class); + conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2); + // Create Mock RM + MockRM rm = new MockRM(getCapacityConfiguration(conf)); + CapacityScheduler sch = (CapacityScheduler) rm.getResourceScheduler(); + // add node + Resource newResource = Resource.newInstance(4 * GB, 1); + RMNode node = MockNodes.newNodeInfo(0, newResource, 1, "127.0.0.1"); + SchedulerEvent addNode = new NodeAddedSchedulerEvent(node); + sch.handle(addNode); + // create appid + ApplicationId appId = BuilderUtils.newApplicationId(100, 1); + ApplicationAttemptId appAttemptId = + BuilderUtils.newApplicationAttemptId(appId, 1); + + RMAppAttemptMetrics attemptMetric = + new RMAppAttemptMetrics(appAttemptId, rm.getRMContext()); + RMAppImpl app = mock(RMAppImpl.class); + when(app.getApplicationId()).thenReturn(appId); + RMAppAttemptImpl attempt = mock(RMAppAttemptImpl.class); + Container container = mock(Container.class); + when(attempt.getMasterContainer()).thenReturn(container); + ApplicationSubmissionContext submissionContext = + mock(ApplicationSubmissionContext.class); + when(attempt.getSubmissionContext()).thenReturn(submissionContext); + when(attempt.getAppAttemptId()).thenReturn(appAttemptId); + when(attempt.getRMAppAttemptMetrics()).thenReturn(attemptMetric); + when(app.getCurrentAppAttempt()).thenReturn(attempt); + + rm.getRMContext().getRMApps().put(appId, app); + // Add application + SchedulerEvent addAppEvent = + new AppAddedSchedulerEvent(appId, "a1", "user"); + sch.handle(addAppEvent); + // Add application attempt + SchedulerEvent addAttemptEvent = + new AppAttemptAddedSchedulerEvent(appAttemptId, false); + sch.handle(addAttemptEvent); + // get Queues + CSQueue queueA1 = sch.getQueue("a1"); + CSQueue queueB = sch.getQueue("b"); + CSQueue queueB1 = sch.getQueue("b1"); + + // add Running rm container and simulate live containers to a1 + ContainerId newContainerId = ContainerId.newContainerId(appAttemptId, 2); + RMContainerImpl rmContainer = mock(RMContainerImpl.class); + when(rmContainer.getState()).thenReturn(RMContainerState.RUNNING); + Container container2 = mock(Container.class); + when(rmContainer.getContainer()).thenReturn(container2); + Resource resource = Resource.newInstance(1024, 1); + when(container2.getResource()).thenReturn(resource); + when(rmContainer.getExecutionType()).thenReturn(ExecutionType.GUARANTEED); + when(container2.getNodeId()).thenReturn(node.getNodeID()); + when(container2.getId()).thenReturn(newContainerId); + when(rmContainer.getNodeLabelExpression()) + .thenReturn(RMNodeLabelsManager.NO_LABEL); + when(rmContainer.getContainerId()).thenReturn(newContainerId); + sch.getApplicationAttempt(appAttemptId).getLiveContainersMap() + .put(newContainerId, rmContainer); + QueueMetrics queueA1M = queueA1.getMetrics(); + queueA1M.incrPendingResources("user1", 1, resource); + queueA1M.allocateResources("user1", resource); + // remove attempt + sch.handle(new AppAttemptRemovedSchedulerEvent(appAttemptId, + RMAppAttemptState.KILLED, true)); + // Move application to queue b1 + sch.moveApplication(appId, "b1"); + // Check queue metrics after move + Assert.assertEquals(0, queueA1.getNumApplications()); + Assert.assertEquals(1, queueB.getNumApplications()); + Assert.assertEquals(0, queueB1.getNumApplications()); + + // Release attempt add event + ApplicationAttemptId appAttemptId2 = + BuilderUtils.newApplicationAttemptId(appId, 2); + SchedulerEvent addAttemptEvent2 = + new AppAttemptAddedSchedulerEvent(appAttemptId2, true); + sch.handle(addAttemptEvent2); + + // Check metrics after attempt added + Assert.assertEquals(0, queueA1.getNumApplications()); + Assert.assertEquals(1, queueB.getNumApplications()); + Assert.assertEquals(1, queueB1.getNumApplications()); + + + QueueMetrics queueB1M = queueB1.getMetrics(); + QueueMetrics queueBM = queueB.getMetrics(); + // Verify allocation MB of current state + Assert.assertEquals(0, queueA1M.getAllocatedMB()); + Assert.assertEquals(0, queueA1M.getAllocatedVirtualCores()); + Assert.assertEquals(1024, queueB1M.getAllocatedMB()); + Assert.assertEquals(1, queueB1M.getAllocatedVirtualCores()); + + // remove attempt + sch.handle(new AppAttemptRemovedSchedulerEvent(appAttemptId2, + RMAppAttemptState.FINISHED, false)); + + Assert.assertEquals(0, queueA1M.getAllocatedMB()); + Assert.assertEquals(0, queueA1M.getAllocatedVirtualCores()); + Assert.assertEquals(0, queueB1M.getAllocatedMB()); + Assert.assertEquals(0, queueB1M.getAllocatedVirtualCores()); + + verifyQueueMetrics(queueB1M); + verifyQueueMetrics(queueBM); + // Verify queue A1 metrics + verifyQueueMetrics(queueA1M); + rm.close(); + } + + private void verifyQueueMetrics(QueueMetrics queue) { + Assert.assertEquals(0, queue.getPendingMB()); + Assert.assertEquals(0, queue.getActiveUsers()); + Assert.assertEquals(0, queue.getActiveApps()); + Assert.assertEquals(0, queue.getAppsPending()); + Assert.assertEquals(0, queue.getAppsRunning()); + Assert.assertEquals(0, queue.getAllocatedMB()); + Assert.assertEquals(0, queue.getAllocatedVirtualCores()); + } + + private Configuration getCapacityConfiguration(Configuration config) { + CapacitySchedulerConfiguration conf = + new CapacitySchedulerConfiguration(config); + + // Define top-level queues + conf.setQueues(CapacitySchedulerConfiguration.ROOT, + new String[] {"a", "b"}); + conf.setCapacity(A, 50); + conf.setCapacity(B, 50); + conf.setQueues(A, new String[] {"a1", "a2"}); + conf.setCapacity(A1, 50); + conf.setCapacity(A2, 50); + conf.setQueues(B, new String[] {"b1"}); + conf.setCapacity(B1, 100); + return conf; + } + @Test public void testKillAllAppsInQueue() throws Exception { MockRM rm = setUpMove(); From 4ebe8a6a237258de9a7d8b041d78249bd3cca7a6 Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Wed, 8 Mar 2017 10:27:57 -0600 Subject: [PATCH 031/188] MAPREDUCE-6859. hadoop-mapreduce-client-jobclient.jar sets a main class that isn't in the JAR. Contributed by Daniel Templeton --- .../hadoop-mapreduce-client-jobclient/pom.xml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml index 5cecebbf693..1747f5989d9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml @@ -143,17 +143,15 @@ **/hdfs-site.xml + + + org.apache.hadoop.test.MapredTestDriver + + test-compile - - - - org.apache.hadoop.test.MapredTestDriver - - - org.apache.maven.plugins From 5addacb1e301991a8285a221c726f66330cd6d08 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Wed, 8 Mar 2017 08:47:38 -0800 Subject: [PATCH 032/188] HDFS-11152. Start erasure coding policy ID number from 1 instead of 0 to void potential unexpected errors. Contributed by SammiChen. --- .../org/apache/hadoop/hdfs/protocol/HdfsConstants.java | 10 +++++----- .../hdfs/server/namenode/FSImageFormatPBINode.java | 3 +++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java index a9f18392450..d2209a49952 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java @@ -144,11 +144,11 @@ public final class HdfsConstants { ALL, LIVE, DEAD, DECOMMISSIONING, ENTERING_MAINTENANCE } - public static final byte RS_6_3_POLICY_ID = 0; - public static final byte RS_3_2_POLICY_ID = 1; - public static final byte RS_6_3_LEGACY_POLICY_ID = 2; - public static final byte XOR_2_1_POLICY_ID = 3; - public static final byte RS_10_4_POLICY_ID = 4; + public static final byte RS_6_3_POLICY_ID = 1; + public static final byte RS_3_2_POLICY_ID = 2; + public static final byte RS_6_3_LEGACY_POLICY_ID = 3; + public static final byte XOR_2_1_POLICY_ID = 4; + public static final byte RS_10_4_POLICY_ID = 5; /* Hidden constructor */ protected HdfsConstants() { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java index 0ceae7816d7..17b1da7422f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java @@ -342,6 +342,9 @@ public final class FSImageFormatPBINode { for (int i = 0; i < bp.size(); ++i) { BlockProto b = bp.get(i); if (isStriped) { + Preconditions.checkState(ecPolicy.getId() > 0, + "File with ID " + n.getId() + + " has an invalid erasure coding policy ID " + ecPolicy.getId()); blocks[i] = new BlockInfoStriped(PBHelperClient.convert(b), ecPolicy); } else { blocks[i] = new BlockInfoContiguous(PBHelperClient.convert(b), From 98142d2f722e82d57b0e2bae6276f7c17fd99598 Mon Sep 17 00:00:00 2001 From: John Zhuge Date: Mon, 6 Mar 2017 11:14:33 -0800 Subject: [PATCH 033/188] Revert "HADOOP-13606 swift FS to add a service load metadata file. Contributed by Steve Loughran" This reverts commit 53a12fa721bb431f7d481aac7d245c93efb56153. --- .../src/main/resources/core-default.xml | 6 ++++++ .../services/org.apache.hadoop.fs.FileSystem | 16 ---------------- 2 files changed, 6 insertions(+), 16 deletions(-) delete mode 100644 hadoop-tools/hadoop-openstack/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 52b58eddb1e..f742ba824f9 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -849,6 +849,12 @@ + + fs.swift.impl + org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem + The implementation class of the OpenStack Swift Filesystem + + fs.automatic.close true diff --git a/hadoop-tools/hadoop-openstack/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem b/hadoop-tools/hadoop-openstack/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem deleted file mode 100644 index 649ea311a8e..00000000000 --- a/hadoop-tools/hadoop-openstack/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem From 287ba4ffa66212c02e1b1edc8fca53f6368a9efc Mon Sep 17 00:00:00 2001 From: Robert Kanter Date: Wed, 8 Mar 2017 10:45:33 -0800 Subject: [PATCH 034/188] YARN-6297. TestAppLogAggregatorImp.verifyFilesUploaded() should check # of filed uploaded with that of files expected (haibochen via rkanter) --- .../logaggregation/TestAppLogAggregatorImpl.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestAppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestAppLogAggregatorImpl.java index 2602d55603c..17d527a7a5d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestAppLogAggregatorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestAppLogAggregatorImpl.java @@ -146,7 +146,7 @@ public class TestAppLogAggregatorImpl { verifyLogAggregationWithExpectedFiles2DeleteAndUpload(applicationId, containerId, logRententionSec, recoveredLogInitedTimeMillis, - logFiles, new HashSet()); + logFiles, logFiles); } @Test @@ -170,7 +170,7 @@ public class TestAppLogAggregatorImpl { final long week = 7 * 24 * 60 * 60; final long recoveredLogInitedTimeMillis = System.currentTimeMillis() - - 2*week; + 2 * week * 1000; verifyLogAggregationWithExpectedFiles2DeleteAndUpload( applicationId, containerId, week, recoveredLogInitedTimeMillis, logFiles, new HashSet()); @@ -257,7 +257,7 @@ public class TestAppLogAggregatorImpl { Set filesExpected) { final String errMsgPrefix = "The set of files uploaded are not the same " + "as expected"; - if(filesUploaded.size() != filesUploaded.size()) { + if(filesUploaded.size() != filesExpected.size()) { fail(errMsgPrefix + ": actual size: " + filesUploaded.size() + " vs " + "expected size: " + filesExpected.size()); } @@ -413,7 +413,7 @@ public class TestAppLogAggregatorImpl { FileContext lfs, long recoveredLogInitedTime) throws IOException { super(dispatcher, deletionService, conf, appId, ugi, nodeId, dirsHandler, remoteNodeLogFileForApp, appAcls, - logAggregationContext, context, lfs, recoveredLogInitedTime); + logAggregationContext, context, lfs, -1, recoveredLogInitedTime); this.applicationId = appId; this.deletionService = deletionService; From 241c1cc05b71f8b719a85c06e3df930639630726 Mon Sep 17 00:00:00 2001 From: Jian He Date: Wed, 8 Mar 2017 10:48:27 -0800 Subject: [PATCH 035/188] HADOOP-14062. ApplicationMasterProtocolPBClientImpl.allocate fails with EOFException when RPC privacy is enabled. Contributed by Steven Rand --- .../java/org/apache/hadoop/ipc/Client.java | 4 +++- .../yarn/client/api/impl/TestAMRMClient.java | 24 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java index 70b902c7bb9..c0a5be95f9d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java @@ -1768,7 +1768,9 @@ public class Client implements AutoCloseable { } void setSaslClient(SaslRpcClient client) throws IOException { - setInputStream(client.getInputStream(in)); + // Wrap the input stream in a BufferedInputStream to fill the buffer + // before reading its length (HADOOP-14062). + setInputStream(new BufferedInputStream(client.getInputStream(in))); setOutputStream(client.getOutputStream(out)); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java index 43c02710f8c..a52963a2a87 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java @@ -137,6 +137,11 @@ public class TestAMRMClient { // set the minimum allocation so that resource decrease can go under 1024 conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512); conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1); + createClientAndCluster(conf); + } + + private static void createClientAndCluster(Configuration conf) + throws Exception { yarnCluster = new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1); yarnCluster.init(conf); yarnCluster.start(); @@ -861,6 +866,25 @@ public class TestAMRMClient { initAMRMClientAndTest(false); } + @Test (timeout=60000) + public void testAMRMClientWithSaslEncryption() throws Exception { + conf.set("hadoop.rpc.protection", "privacy"); + // we have to create a new instance of MiniYARNCluster to avoid SASL qop + // mismatches between client and server + tearDown(); + createClientAndCluster(conf); + startApp(); + initAMRMClientAndTest(false); + + // recreate the original MiniYARNCluster and YarnClient for other tests + conf.unset("hadoop.rpc.protection"); + tearDown(); + createClientAndCluster(conf); + // unless we start an application the cancelApp() method will fail when + // it runs after this test + startApp(); + } + @Test (timeout=60000) public void testAMRMClientAllocReqId() throws YarnException, IOException { initAMRMClientAndTest(true); From 2be8947d12714c49ef7a90de82a351d086b435b6 Mon Sep 17 00:00:00 2001 From: Jian He Date: Wed, 8 Mar 2017 13:20:01 -0800 Subject: [PATCH 036/188] Revert "HADOOP-14062. ApplicationMasterProtocolPBClientImpl.allocate fails with EOFException when RPC privacy is enabled. Contributed by Steven Rand" This reverts commit 241c1cc05b71f8b719a85c06e3df930639630726. --- .../java/org/apache/hadoop/ipc/Client.java | 4 +--- .../yarn/client/api/impl/TestAMRMClient.java | 24 ------------------- 2 files changed, 1 insertion(+), 27 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java index c0a5be95f9d..70b902c7bb9 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java @@ -1768,9 +1768,7 @@ public class Client implements AutoCloseable { } void setSaslClient(SaslRpcClient client) throws IOException { - // Wrap the input stream in a BufferedInputStream to fill the buffer - // before reading its length (HADOOP-14062). - setInputStream(new BufferedInputStream(client.getInputStream(in))); + setInputStream(client.getInputStream(in)); setOutputStream(client.getOutputStream(out)); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java index a52963a2a87..43c02710f8c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java @@ -137,11 +137,6 @@ public class TestAMRMClient { // set the minimum allocation so that resource decrease can go under 1024 conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512); conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1); - createClientAndCluster(conf); - } - - private static void createClientAndCluster(Configuration conf) - throws Exception { yarnCluster = new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1); yarnCluster.init(conf); yarnCluster.start(); @@ -866,25 +861,6 @@ public class TestAMRMClient { initAMRMClientAndTest(false); } - @Test (timeout=60000) - public void testAMRMClientWithSaslEncryption() throws Exception { - conf.set("hadoop.rpc.protection", "privacy"); - // we have to create a new instance of MiniYARNCluster to avoid SASL qop - // mismatches between client and server - tearDown(); - createClientAndCluster(conf); - startApp(); - initAMRMClientAndTest(false); - - // recreate the original MiniYARNCluster and YarnClient for other tests - conf.unset("hadoop.rpc.protection"); - tearDown(); - createClientAndCluster(conf); - // unless we start an application the cancelApp() method will fail when - // it runs after this test - startApp(); - } - @Test (timeout=60000) public void testAMRMClientAllocReqId() throws YarnException, IOException { initAMRMClientAndTest(true); From d7762a55113a529abd6f4ecb8e6d9b0a84b56e08 Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Wed, 8 Mar 2017 16:46:09 -0600 Subject: [PATCH 037/188] YARN-6165. Intra-queue preemption occurs even when preemption is turned off for a specific queue. Contributed by Eric Payne --- .../IntraQueueCandidatesSelector.java | 5 ++ ...nalCapacityPreemptionPolicyIntraQueue.java | 55 +++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/IntraQueueCandidatesSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/IntraQueueCandidatesSelector.java index 4f2b272cf7a..289041499d7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/IntraQueueCandidatesSelector.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/IntraQueueCandidatesSelector.java @@ -112,6 +112,11 @@ public class IntraQueueCandidatesSelector extends PreemptionCandidatesSelector { continue; } + // Don't preempt if disabled for this queue. + if (leafQueue.getPreemptionDisabled()) { + continue; + } + // 5. Calculate the resource to obtain per partition Map resToObtainByPartition = fifoPreemptionComputePlugin .getResourceDemandFromAppsPerQueue(queueName, partition); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueue.java index 19fb0d293d0..bf83e1c7387 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueue.java @@ -105,6 +105,61 @@ public class TestProportionalCapacityPreemptionPolicyIntraQueue getAppAttemptId(3)))); } + @Test + public void testNoIntraQueuePreemptionWithPreemptionDisabledOnQueues() + throws IOException { + /** + * This test has the same configuration as testSimpleIntraQueuePreemption + * except that preemption is disabled specifically for each queue. The + * purpose is to test that disabling preemption on a specific queue will + * avoid intra-queue preemption. + */ + conf.setPreemptionDisabled("root.a", true); + conf.setPreemptionDisabled("root.b", true); + conf.setPreemptionDisabled("root.c", true); + conf.setPreemptionDisabled("root.d", true); + + String labelsConfig = "=100,true;"; + String nodesConfig = // n1 has no label + "n1= res=100"; + String queuesConfig = + // guaranteed,max,used,pending,reserved + "root(=[100 100 80 120 0]);" + // root + "-a(=[11 100 11 50 0]);" + // a + "-b(=[40 100 38 60 0]);" + // b + "-c(=[20 100 10 10 0]);" + // c + "-d(=[29 100 20 0 0])"; // d + + String appsConfig = + // queueName\t(priority,resource,host,expression,#repeat,reserved, + // pending) + "a\t" // app1 in a + + "(1,1,n1,,6,false,25);" + // app1 a + "a\t" // app2 in a + + "(1,1,n1,,5,false,25);" + // app2 a + "b\t" // app3 in b + + "(4,1,n1,,34,false,20);" + // app3 b + "b\t" // app4 in b + + "(4,1,n1,,2,false,10);" + // app4 b + "b\t" // app4 in b + + "(5,1,n1,,1,false,10);" + // app5 b + "b\t" // app4 in b + + "(6,1,n1,,1,false,10);" + // app6 in b + "c\t" // app1 in a + + "(1,1,n1,,10,false,10);" + "d\t" // app7 in c + + "(1,1,n1,,20,false,0)"; + + buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig); + policy.editSchedule(); + + verify(mDisp, times(0)).handle(argThat( + new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor( + getAppAttemptId(4)))); + verify(mDisp, times(0)).handle(argThat( + new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor( + getAppAttemptId(3)))); + } + @Test public void testNoPreemptionForSamePriorityApps() throws IOException { /** From 5ca6ef0c268b1acb3abf12505b9ead6fe7e38a23 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Wed, 8 Mar 2017 15:36:19 -0800 Subject: [PATCH 038/188] HDFS-10983. OIV tool should make an EC file explicit. Contributed by Manoj Govindassamy. --- .../server/namenode/FSImageFormatPBINode.java | 1 + .../OfflineImageReconstructor.java | 4 + .../offlineImageViewer/PBImageXmlWriter.java | 15 ++- .../hdfs/server/namenode/TestFSImage.java | 1 + .../TestOfflineImageViewer.java | 99 ++++++++++++++++++- 5 files changed, 112 insertions(+), 8 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java index 17b1da7422f..ef334f7fd4c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java @@ -332,6 +332,7 @@ public final class FSImageFormatPBINode { BlockType blockType = PBHelperClient.convert(f.getBlockType()); LoaderContext state = parent.getLoaderContext(); boolean isStriped = f.hasErasureCodingPolicyID(); + assert ((!isStriped) || (isStriped && !f.hasReplication())); Short replication = (!isStriped ? (short) f.getReplication() : null); ErasureCodingPolicy ecPolicy = isStriped ? ErasureCodingPolicyManager.getPolicyByPolicyID( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java index ed348d39033..e80f4d43e68 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java @@ -647,6 +647,10 @@ class OfflineImageReconstructor { break; case "STRIPED": bld.setBlockType(HdfsProtos.BlockTypeProto.STRIPED); + ival = node.removeChildInt(INODE_SECTION_EC_POLICY_ID); + if (ival != null) { + bld.setErasureCodingPolicyID(ival); + } break; default: throw new IOException("INode XML found with unknown " + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java index f8734cbe293..5a42a6b7b02 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java @@ -40,7 +40,6 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheD import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolInfoProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto; -import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockTypeProto; import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos; import org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode; import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.SectionName; @@ -59,6 +58,7 @@ import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SecretManagerSection; import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SnapshotDiffSection; import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SnapshotSection; import org.apache.hadoop.hdfs.server.namenode.FsImageProto.StringTableSection; +import org.apache.hadoop.hdfs.server.namenode.INodeFile; import org.apache.hadoop.hdfs.util.XMLUtils; import org.apache.hadoop.util.LimitInputStream; import com.google.common.collect.ImmutableList; @@ -132,6 +132,8 @@ public final class PBImageXmlWriter { public static final String INODE_SECTION_STORAGE_POLICY_ID = "storagePolicyId"; public static final String INODE_SECTION_BLOCK_TYPE = "blockType"; + public static final String INODE_SECTION_EC_POLICY_ID = + "erasureCodingPolicyId"; public static final String INODE_SECTION_NS_QUOTA = "nsquota"; public static final String INODE_SECTION_DS_QUOTA = "dsquota"; public static final String INODE_SECTION_TYPE_QUOTA = "typeQuota"; @@ -472,8 +474,12 @@ public final class PBImageXmlWriter { } private void dumpINodeFile(INodeSection.INodeFile f) { - o(SECTION_REPLICATION, f.getReplication()) - .o(INODE_SECTION_MTIME, f.getModificationTime()) + if (f.hasErasureCodingPolicyID()) { + o(SECTION_REPLICATION, INodeFile.DEFAULT_REPL_FOR_STRIPED_BLOCKS); + } else { + o(SECTION_REPLICATION, f.getReplication()); + } + o(INODE_SECTION_MTIME, f.getModificationTime()) .o(INODE_SECTION_ATIME, f.getAccessTime()) .o(INODE_SECTION_PREFERRED_BLOCK_SIZE, f.getPreferredBlockSize()) .o(INODE_SECTION_PERMISSION, dumpPermission(f.getPermission())); @@ -495,8 +501,9 @@ public final class PBImageXmlWriter { if (f.hasStoragePolicyID()) { o(INODE_SECTION_STORAGE_POLICY_ID, f.getStoragePolicyID()); } - if (f.getBlockType() != BlockTypeProto.CONTIGUOUS) { + if (f.hasErasureCodingPolicyID()) { o(INODE_SECTION_BLOCK_TYPE, f.getBlockType().name()); + o(INODE_SECTION_EC_POLICY_ID, f.getErasureCodingPolicyID()); } if (f.hasFileUC()) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java index ae154917867..57416ed16d7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java @@ -223,6 +223,7 @@ public class TestFSImage { // blocks to/from legacy fsimage assertEquals(3, fileByLoaded.getBlocks().length); assertEquals(preferredBlockSize, fileByLoaded.getPreferredBlockSize()); + assertEquals(file.getFileReplication(), fileByLoaded.getFileReplication()); if (isUC) { assertEquals(client, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java index b587017b91b..e52d5365981 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java @@ -76,11 +76,13 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.protocol.BlockType; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; +import org.apache.hadoop.hdfs.server.namenode.INodeFile; import org.apache.hadoop.hdfs.server.namenode.NameNodeLayoutVersion; import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; import org.apache.hadoop.io.IOUtils; @@ -91,9 +93,8 @@ import org.apache.log4j.Level; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.TemporaryFolder; +import org.xml.sax.Attributes; import org.xml.sax.InputSource; import org.xml.sax.SAXException; import org.xml.sax.helpers.DefaultHandler; @@ -112,7 +113,6 @@ public class TestOfflineImageViewer { // namespace as written to dfs, to be compared with viewer's output final static HashMap writtenFiles = Maps.newHashMap(); static int dirCount = 0; - private static File tempDir; // Create a populated namespace for later testing. Save its contents to a @@ -358,6 +358,96 @@ public class TestOfflineImageViewer { assertEquals(0, status); } + /** + * SAX handler to verify EC Files and their policies. + */ + class ECXMLHandler extends DefaultHandler { + + private boolean isInode = false; + private boolean isAttrRepl = false; + private boolean isAttrName = false; + private boolean isXAttrs = false; + private boolean isAttrECPolicy = false; + private boolean isAttrBlockType = false; + private String currentInodeName; + private String currentECPolicy; + private String currentBlockType; + private String currentRepl; + + @Override + public void startElement(String uri, String localName, String qName, + Attributes attributes) throws SAXException { + super.startElement(uri, localName, qName, attributes); + if (qName.equalsIgnoreCase(PBImageXmlWriter.INODE_SECTION_INODE)) { + isInode = true; + } else if (isInode && !isXAttrs && qName.equalsIgnoreCase( + PBImageXmlWriter.SECTION_NAME)) { + isAttrName = true; + } else if (isInode && qName.equalsIgnoreCase( + PBImageXmlWriter.SECTION_REPLICATION)) { + isAttrRepl = true; + } else if (isInode && + qName.equalsIgnoreCase(PBImageXmlWriter.INODE_SECTION_EC_POLICY_ID)) { + isAttrECPolicy = true; + } else if (isInode && qName.equalsIgnoreCase( + PBImageXmlWriter.INODE_SECTION_BLOCK_TYPE)) { + isAttrBlockType = true; + } else if (isInode && qName.equalsIgnoreCase( + PBImageXmlWriter.INODE_SECTION_XATTRS)) { + isXAttrs = true; + } + } + + @Override + public void endElement(String uri, String localName, String qName) + throws SAXException { + super.endElement(uri, localName, qName); + if (qName.equalsIgnoreCase(PBImageXmlWriter.INODE_SECTION_INODE)) { + if (currentInodeName != null && currentInodeName.length() > 0) { + if (currentBlockType != null && currentBlockType.equalsIgnoreCase( + BlockType.STRIPED.name())) { + Assert.assertEquals("INode '" + + currentInodeName + "' has unexpected EC Policy!", + Byte.parseByte(currentECPolicy), + ErasureCodingPolicyManager.getPolicyByPolicyID( + HdfsConstants.XOR_2_1_POLICY_ID).getId()); + Assert.assertEquals("INode '" + + currentInodeName + "' has unexpected replication!", + currentRepl, + Short.toString(INodeFile.DEFAULT_REPL_FOR_STRIPED_BLOCKS)); + } + } + isInode = false; + currentInodeName = ""; + currentECPolicy = ""; + currentRepl = ""; + } else if (qName.equalsIgnoreCase( + PBImageXmlWriter.INODE_SECTION_XATTRS)) { + isXAttrs = false; + } + } + + @Override + public void characters(char[] ch, int start, int length) + throws SAXException { + super.characters(ch, start, length); + String value = new String(ch, start, length); + if (isAttrName) { + currentInodeName = value; + isAttrName = false; + } else if (isAttrRepl) { + currentRepl = value; + isAttrRepl = false; + } else if (isAttrECPolicy) { + currentECPolicy = value; + isAttrECPolicy = false; + } else if (isAttrBlockType) { + currentBlockType = value; + isAttrBlockType = false; + } + } + } + @Test public void testPBImageXmlWriter() throws IOException, SAXException, ParserConfigurationException { @@ -368,7 +458,8 @@ public class TestOfflineImageViewer { SAXParserFactory spf = SAXParserFactory.newInstance(); SAXParser parser = spf.newSAXParser(); final String xml = output.toString(); - parser.parse(new InputSource(new StringReader(xml)), new DefaultHandler()); + ECXMLHandler ecxmlHandler = new ECXMLHandler(); + parser.parse(new InputSource(new StringReader(xml)), ecxmlHandler); } @Test From 33a38a534110de454662256545a7f4c075d328c8 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Wed, 8 Mar 2017 16:41:44 -0800 Subject: [PATCH 039/188] HDFS-11314. Enforce set of enabled EC policies on the NameNode. --- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 + .../namenode/ErasureCodingPolicyManager.java | 101 ++++++++---- .../server/namenode/FSDirErasureCodingOp.java | 54 +++---- .../server/namenode/FSImageFormatPBINode.java | 2 +- .../hdfs/server/namenode/FSNamesystem.java | 2 +- .../hdfs/server/namenode/INodeFile.java | 4 +- .../org/apache/hadoop/hdfs/tools/ECAdmin.java | 24 ++- .../src/main/resources/hdfs-default.xml | 9 ++ .../src/site/markdown/HDFSErasureCoding.md | 45 ++++-- .../hadoop/cli/TestErasureCodingCLI.java | 3 + .../org/apache/hadoop/hdfs/DFSTestUtil.java | 11 ++ ...estDFSRSDefault10x4StripedInputStream.java | 2 +- ...stDFSRSDefault10x4StripedOutputStream.java | 2 +- ...ult10x4StripedOutputStreamWithFailure.java | 2 +- .../hdfs/TestDFSStripedInputStream.java | 2 + .../hdfs/TestDFSStripedOutputStream.java | 1 + ...TestDFSStripedOutputStreamWithFailure.java | 1 + .../hdfs/TestDFSXORStripedInputStream.java | 2 +- .../hdfs/TestDFSXORStripedOutputStream.java | 2 +- ...tDFSXORStripedOutputStreamWithFailure.java | 2 +- .../hdfs/TestErasureCodingPolicies.java | 41 ++++- .../TestUnsetAndChangeDirectoryEcPolicy.java | 5 +- .../namenode/TestEnabledECPolicies.java | 151 ++++++++++++++++++ .../hdfs/server/namenode/TestFSImage.java | 9 +- .../TestOfflineImageViewer.java | 11 +- .../test/resources/testErasureCodingConf.xml | 21 ++- 26 files changed, 401 insertions(+), 110 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEnabledECPolicies.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 82d6073399a..3fc4980648d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -562,6 +562,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_DATANODE_DISK_CHECK_TIMEOUT_DEFAULT = "10m"; + public static final String DFS_NAMENODE_EC_POLICIES_ENABLED_KEY = "dfs.namenode.ec.policies.enabled"; + public static final String DFS_NAMENODE_EC_POLICIES_ENABLED_DEFAULT = "RS-6-3-64k"; public static final String DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_THREADS_KEY = "dfs.datanode.ec.reconstruction.stripedread.threads"; public static final int DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_THREADS_DEFAULT = 20; public static final String DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_BUFFER_SIZE_KEY = "dfs.datanode.ec.reconstruction.stripedread.buffer.size"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java index a1b22708b20..02cbbdf38fd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java @@ -18,12 +18,17 @@ package org.apache.hadoop.hdfs.server.namenode; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; +import java.util.Arrays; import java.util.Map; import java.util.TreeMap; +import java.util.stream.Collectors; + /** * This manages erasure coding policies predefined and activated in the system. @@ -61,21 +66,52 @@ public final class ErasureCodingPolicyManager { SYS_POLICY4, SYS_POLICY5}; // Supported storage policies for striped EC files - private static final byte[] SUITABLE_STORAGE_POLICIES_FOR_EC_STRIPED_MODE = new byte[] { - HdfsConstants.HOT_STORAGE_POLICY_ID, HdfsConstants.COLD_STORAGE_POLICY_ID, - HdfsConstants.ALLSSD_STORAGE_POLICY_ID }; - + private static final byte[] SUITABLE_STORAGE_POLICIES_FOR_EC_STRIPED_MODE = + new byte[]{ + HdfsConstants.HOT_STORAGE_POLICY_ID, + HdfsConstants.COLD_STORAGE_POLICY_ID, + HdfsConstants.ALLSSD_STORAGE_POLICY_ID}; /** - * All active policies maintained in NN memory for fast querying, + * All supported policies maintained in NN memory for fast querying, * identified and sorted by its name. */ - private final Map activePoliciesByName; + private static final Map SYSTEM_POLICIES_BY_NAME; - ErasureCodingPolicyManager() { - - this.activePoliciesByName = new TreeMap<>(); + static { + // Create a hashmap of all available policies for quick lookup by name + SYSTEM_POLICIES_BY_NAME = new TreeMap<>(); for (ErasureCodingPolicy policy : SYS_POLICIES) { - activePoliciesByName.put(policy.getName(), policy); + SYSTEM_POLICIES_BY_NAME.put(policy.getName(), policy); + } + } + + /** + * All enabled policies maintained in NN memory for fast querying, + * identified and sorted by its name. + */ + private final Map enabledPoliciesByName; + + ErasureCodingPolicyManager(Configuration conf) { + // Populate the list of enabled policies from configuration + final String[] policyNames = conf.getTrimmedStrings( + DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_DEFAULT); + this.enabledPoliciesByName = new TreeMap<>(); + for (String policyName : policyNames) { + ErasureCodingPolicy ecPolicy = SYSTEM_POLICIES_BY_NAME.get(policyName); + if (ecPolicy == null) { + String sysPolicies = Arrays.asList(SYS_POLICIES).stream() + .map(ErasureCodingPolicy::getName) + .collect(Collectors.joining(", ")); + String msg = String.format("EC policy %s specified at %s is not a " + + "valid policy. Please choose from list of available policies: " + + "[%s]", + policyName, + DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + sysPolicies); + throw new IllegalArgumentException(msg); + } + enabledPoliciesByName.put(ecPolicy.getName(), ecPolicy); } /** @@ -104,10 +140,10 @@ public final class ErasureCodingPolicyManager { } /** - * Get system-wide policy by policy ID. - * @return ecPolicy + * Get a policy by policy ID. + * @return ecPolicy, or null if not found */ - public static ErasureCodingPolicy getPolicyByPolicyID(byte id) { + public static ErasureCodingPolicy getPolicyByID(byte id) { for (ErasureCodingPolicy policy : SYS_POLICIES) { if (policy.getId() == id) { return policy; @@ -117,36 +153,33 @@ public final class ErasureCodingPolicyManager { } /** - * Get all policies that's available to use. + * Get a policy by policy name. + * @return ecPolicy, or null if not found + */ + public static ErasureCodingPolicy getPolicyByName(String name) { + return SYSTEM_POLICIES_BY_NAME.get(name); + } + + /** + * Get the set of enabled policies. * @return all policies */ - public ErasureCodingPolicy[] getPolicies() { + public ErasureCodingPolicy[] getEnabledPolicies() { ErasureCodingPolicy[] results = - new ErasureCodingPolicy[activePoliciesByName.size()]; - return activePoliciesByName.values().toArray(results); + new ErasureCodingPolicy[enabledPoliciesByName.size()]; + return enabledPoliciesByName.values().toArray(results); } /** - * Get the policy specified by the policy name. + * Get enabled policy by policy name. */ - public ErasureCodingPolicy getPolicyByName(String name) { - return activePoliciesByName.get(name); + public ErasureCodingPolicy getEnabledPolicyByName(String name) { + return enabledPoliciesByName.get(name); } /** - * Get the policy specified by the policy ID. - */ - public ErasureCodingPolicy getPolicyByID(byte id) { - for (ErasureCodingPolicy policy : activePoliciesByName.values()) { - if (policy.getId() == id) { - return policy; - } - } - return null; - } - - /** - * @return True if given policy is be suitable for striped EC Files. + * @return if the specified storage policy ID is suitable for striped EC + * files. */ public static boolean checkStoragePolicySuitableForECStripedMode( byte storagePolicyID) { @@ -164,6 +197,6 @@ public final class ErasureCodingPolicyManager { * Clear and clean up. */ public void clear() { - activePoliciesByName.clear(); + enabledPoliciesByName.clear(); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java index 30a7b8bbe64..50843c87066 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java @@ -23,9 +23,10 @@ import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.FileNotFoundException; import java.io.IOException; -import java.util.ArrayList; +import java.util.Arrays; import java.util.EnumSet; import java.util.List; +import java.util.stream.Collectors; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; @@ -34,6 +35,7 @@ import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.fs.XAttr; import org.apache.hadoop.fs.XAttrSetFlag; import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.XAttrHelper; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; @@ -82,11 +84,23 @@ final class FSDirErasureCodingOp { fsd.writeLock(); try { ErasureCodingPolicy ecPolicy = fsn.getErasureCodingPolicyManager() - .getPolicyByName(ecPolicyName); + .getEnabledPolicyByName(ecPolicyName); if (ecPolicy == null) { - throw new HadoopIllegalArgumentException("Policy '" + - ecPolicyName + "' does not match any supported erasure coding " + - "policies."); + final String sysPolicies = + Arrays.asList( + fsn.getErasureCodingPolicyManager().getEnabledPolicies()) + .stream() + .map(ErasureCodingPolicy::getName) + .collect(Collectors.joining(", ")); + final String message = String.format("Policy '%s' does not match any " + + "enabled erasure" + + " coding policies: [%s]. The set of enabled erasure coding " + + "policies can be configured at '%s'.", + ecPolicyName, + sysPolicies, + DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY + ); + throw new HadoopIllegalArgumentException(message); } iip = fsd.resolvePath(pc, src, DirOp.WRITE_LINK); // Write access is required to set erasure coding policy @@ -118,26 +132,6 @@ final class FSDirErasureCodingOp { "for a file " + src); } - // Check that the EC policy is one of the active policies. - boolean validPolicy = false; - ErasureCodingPolicy[] activePolicies = - FSDirErasureCodingOp.getErasureCodingPolicies(fsd.getFSNamesystem()); - for (ErasureCodingPolicy activePolicy : activePolicies) { - if (activePolicy.equals(ecPolicy)) { - validPolicy = true; - break; - } - } - if (!validPolicy) { - List ecPolicyNames = new ArrayList(); - for (ErasureCodingPolicy activePolicy : activePolicies) { - ecPolicyNames.add(activePolicy.getName()); - } - throw new HadoopIllegalArgumentException("Policy [ " + - ecPolicy.getName() + " ] does not match any of the " + - "supported policies. Please select any one of " + ecPolicyNames); - } - final XAttr ecXAttr; DataOutputStream dOut = null; try { @@ -291,7 +285,7 @@ final class FSDirErasureCodingOp { static ErasureCodingPolicy[] getErasureCodingPolicies(final FSNamesystem fsn) throws IOException { assert fsn.hasReadLock(); - return fsn.getErasureCodingPolicyManager().getPolicies(); + return fsn.getErasureCodingPolicyManager().getEnabledPolicies(); } private static ErasureCodingPolicy getErasureCodingPolicyForPath( @@ -307,8 +301,8 @@ final class FSDirErasureCodingOp { } if (inode.isFile()) { byte id = inode.asFile().getErasureCodingPolicyID(); - return id < 0 ? null : fsd.getFSNamesystem(). - getErasureCodingPolicyManager().getPolicyByID(id); + return id < 0 ? null : + ErasureCodingPolicyManager.getPolicyByID(id); } // We don't allow setting EC policies on paths with a symlink. Thus // if a symlink is encountered, the dir shouldn't have EC policy. @@ -323,8 +317,8 @@ final class FSDirErasureCodingOp { ByteArrayInputStream bIn = new ByteArrayInputStream(xattr.getValue()); DataInputStream dIn = new DataInputStream(bIn); String ecPolicyName = WritableUtils.readString(dIn); - return fsd.getFSNamesystem().getErasureCodingPolicyManager(). - getPolicyByName(ecPolicyName); + return ErasureCodingPolicyManager + .getPolicyByName(ecPolicyName); } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java index ef334f7fd4c..eab728283b3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java @@ -335,7 +335,7 @@ public final class FSImageFormatPBINode { assert ((!isStriped) || (isStriped && !f.hasReplication())); Short replication = (!isStriped ? (short) f.getReplication() : null); ErasureCodingPolicy ecPolicy = isStriped ? - ErasureCodingPolicyManager.getPolicyByPolicyID( + ErasureCodingPolicyManager.getPolicyByID( (byte) f.getErasureCodingPolicyID()) : null; Byte ecPolicyID = (isStriped ? ecPolicy.getId() : null); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 03277af446a..062896ce4f7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -838,7 +838,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, this.dir = new FSDirectory(this, conf); this.snapshotManager = new SnapshotManager(dir); this.cacheManager = new CacheManager(this, conf, blockManager); - this.ecPolicyManager = new ErasureCodingPolicyManager(); + this.ecPolicyManager = new ErasureCodingPolicyManager(conf); this.topConf = new TopConf(conf); this.auditLoggers = initAuditLoggers(conf); this.isDefaultAuditLogger = auditLoggers.size() == 1 && diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java index 19480e45031..2b0e0adea2a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java @@ -192,7 +192,7 @@ public class INodeFile extends INodeWithAdditionalFields Preconditions.checkArgument(replication == null && erasureCodingPolicyID != null); Preconditions.checkArgument(ErasureCodingPolicyManager - .getPolicyByPolicyID(erasureCodingPolicyID) != null, + .getPolicyByID(erasureCodingPolicyID) != null, "Could not find EC policy with ID 0x" + StringUtils .byteToHexString(erasureCodingPolicyID)); layoutRedundancy |= BLOCK_TYPE_MASK_STRIPED; @@ -514,7 +514,7 @@ public class INodeFile extends INodeWithAdditionalFields } ErasureCodingPolicy ecPolicy = - ErasureCodingPolicyManager.getPolicyByPolicyID( + ErasureCodingPolicyManager.getPolicyByID( getErasureCodingPolicyID()); Preconditions.checkNotNull(ecPolicy, "Could not find EC policy with ID 0x" + StringUtils.byteToHexString(getErasureCodingPolicyID())); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java index 548f754a1eb..52f75342173 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java @@ -20,6 +20,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.tools.TableListing; @@ -79,7 +80,7 @@ public class ECAdmin extends Configured implements Tool { } } - /** Command to list the set of available erasure coding policies */ + /** Command to list the set of enabled erasure coding policies. */ private static class ListECPoliciesCommand implements AdminHelper.Command { @Override @@ -95,7 +96,9 @@ public class ECAdmin extends Configured implements Tool { @Override public String getLongUsage() { return getShortUsage() + "\n" + - "Get the list of supported erasure coding policies.\n"; + "Get the list of enabled erasure coding policies.\n" + + "Policies can be enabled on the NameNode via `" + + DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY + "`.\n"; } @Override @@ -109,10 +112,19 @@ public class ECAdmin extends Configured implements Tool { try { Collection policies = dfs.getAllErasureCodingPolicies(); - System.out.println("Erasure Coding Policies:"); - for (ErasureCodingPolicy policy : policies) { - if (policy != null) { - System.out.println("\t" + policy.getName()); + if (policies.isEmpty()) { + System.out.println("No erasure coding policies are enabled on the " + + "cluster."); + System.out.println("The set of enabled policies can be " + + "configured at '" + + DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY + "' on the " + + "NameNode."); + } else { + System.out.println("Erasure Coding Policies:"); + for (ErasureCodingPolicy policy : policies) { + if (policy != null) { + System.out.println("\t" + policy.getName()); + } } } } catch (IOException e) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 36b93b7e949..fa5d814eaf8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -2928,6 +2928,15 @@ + + dfs.namenode.ec.policies.enabled + RS-6-3-64k + Comma-delimited list of enabled erasure coding policies. + The NameNode will enforce this when setting an erasure coding policy + on a directory. + + + dfs.datanode.ec.reconstruction.stripedread.timeout.millis 5000 diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md index 36fb61dfabf..9d9f2ce3c6e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md @@ -56,19 +56,26 @@ Architecture 3. _Transfer the generated data blocks to target nodes:_ Once decoding is finished, the recovered blocks are transferred to target DataNodes. - * **ErasureCoding policy** - To accommodate heterogeneous workloads, we allow files and directories in an HDFS cluster to have different replication and EC policies. - Information on how to encode/decode a file is encapsulated in an ErasureCodingPolicy class. Each policy is defined by the following 2 pieces of information: + * **Erasure coding policies** + To accommodate heterogeneous workloads, we allow files and directories in an HDFS cluster to have different replication and erasure coding policies. + The erasure coding policy encapsulates how to encode/decode a file. Each policy is defined by the following pieces of information: - 1. _The ECSchema:_ This includes the numbers of data and parity blocks in an EC group (e.g., 6+3), as well as the codec algorithm (e.g., Reed-Solomon). + 1. _The EC schema:_ This includes the numbers of data and parity blocks in an EC group (e.g., 6+3), as well as the codec algorithm (e.g., Reed-Solomon, XOR). 2. _The size of a striping cell._ This determines the granularity of striped reads and writes, including buffer sizes and encoding work. - Five policies are currently supported: RS-3-2-64k, RS-6-3-64k, RS-10-4-64k, RS-LEGACY-6-3-64k, and XOR-2-1-64k. All with default cell size of 64KB. The system default policy is RS-6-3-64k which use the default schema RS_6_3_SCHEMA with a cell size of 64KB. + Policies are named *codec*-*num data blocks*-*num parity blocks*-*cell size*. Currently, five built-in policies are supported: `RS-3-2-64k`, `RS-6-3-64k`, `RS-10-4-64k`, `RS-LEGACY-6-3-64k`, and `XOR-2-1-64k`. + + By default, only `RS-6-3-64k` is enabled. + + Similar to HDFS storage policies, erasure coding policies are set on a directory. When a file is created, it inherits the EC policy of its nearest ancestor directory. + + Directory-level EC policies only affect new files created within the directory. Once a file has been created, its erasure coding policy can be queried but not changed. If an erasure coded file is renamed to a directory with a different EC policy, the file retains its existing EC policy. Converting a file to a different EC policy requires rewriting its data; do this by copying the file (e.g. via distcp) rather than renaming it. * **Intel ISA-L** - Intel ISA-L stands for Intel Intelligent Storage Acceleration Library. ISA-L is a collection of optimized low-level functions used primarily in storage applications. It includes a fast block Reed-Solomon type erasure codes optimized for Intel AVX and AVX2 instruction sets. - HDFS EC can leverage this open-source library to accelerate encoding and decoding calculation. ISA-L supports most of major operating systems, including Linux and Windows. By default, ISA-L is not enabled in HDFS. + Intel ISA-L stands for Intel Intelligent Storage Acceleration Library. ISA-L is an open-source collection of optimized low-level functions designed for storage applications. It includes fast block Reed-Solomon type erasure codes optimized for Intel AVX and AVX2 instruction sets. + HDFS erasure coding can leverage ISA-L to accelerate encoding and decoding calculation. ISA-L supports most major operating systems, including Linux and Windows. + ISA-L is not enabled by default. See the instructions below for how to enable ISA-L. Deployment ---------- @@ -90,6 +97,10 @@ Deployment ### Configuration keys + The set of enabled erasure coding policies can be configured on the NameNode via `dfs.namenode.ec.policies.enabled`. This restricts what EC policies can be set by clients. It does not affect the behavior of already set file or directory-level EC policies. + + By default, only the `RS-6-3-64k` policy is enabled. Typically, the cluster administrator will configure the set of enabled policies based on the size of the cluster and the desired fault-tolerance properties. For instance, for a cluster with 9 racks, a policy like `RS-10-4-64k` will not preserve rack-level fault-tolerance, and `RS-6-3-64k` or `RS-3-2-64k` might be more appropriate. If the administrator only cares about node-level fault-tolerance, `RS-10-4-64k` would still be appropriate as long as there are at least 14 DataNodes in the cluster. + The codec implementation for Reed-Solomon and XOR can be configured with the following client and DataNode configuration keys: `io.erasurecode.codec.rs.rawcoder` for the default RS codec, `io.erasurecode.codec.rs-legacy.rawcoder` for the legacy RS codec, @@ -106,13 +117,13 @@ Deployment ### Enable Intel ISA-L HDFS native implementation of default RS codec leverages Intel ISA-L library to improve the encoding and decoding calculation. To enable and use Intel ISA-L, there are three steps. - 1. Build ISA-L library. Please refer to the offical site "https://github.com/01org/isa-l/" for detail information. - 2. Build Hadoop with ISA-L support. Please refer to "Intel ISA-L build options" section in "Build instructions for Hadoop"(BUILDING.txt) document. Use -Dbundle.isal to copy the contents of the isal.lib directory into the final tar file. Deploy hadoop with the tar file. Make sure ISA-L library is available on both HDFS client and DataNodes. - 3. Configure the `io.erasurecode.codec.rs.rawcoder` key with value `org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory` on HDFS client and DataNodes. + 1. Build ISA-L library. Please refer to the official site "https://github.com/01org/isa-l/" for detail information. + 2. Build Hadoop with ISA-L support. Please refer to "Intel ISA-L build options" section in "Build instructions for Hadoop" in (BUILDING.txt) in the source code. Use `-Dbundle.isal` to copy the contents of the `isal.lib` directory into the final tar file. Deploy Hadoop with the tar file. Make sure ISA-L is available on HDFS clients and DataNodes. + 3. Configure the `io.erasurecode.codec.rs.rawcoder` key with value `org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory` on HDFS clients and DataNodes. - To check ISA-L library enable state, try "Hadoop checknative" command. It will tell you if ISA-L library is enabled or not. + To verify that ISA-L is correctly detected by Hadoop, run the `hadoop checknative` command. - It also requires three steps to enable the native implementation of XOR codec. The first two steps are the same as the above step 1 and step 2. In step 3, configure the `io.erasurecode.codec.xor.rawcoder` key with `org.apache.hadoop.io.erasurecode.rawcoder.NativeXORRawErasureCoderFactory` on both HDFS client and DataNodes. + To enable the native implementation of the XOR codec, perform the same first two steps as above to build and deploy Hadoop with ISA-L support. Afterwards, configure the `io.erasurecode.codec.xor.rawcoder` key with `org.apache.hadoop.io.erasurecode.rawcoder.NativeXORRawErasureCoderFactory` on both HDFS client and DataNodes. ### Administrative commands @@ -130,20 +141,20 @@ Below are the details about each command. * `[-setPolicy -policy -path ]` - Sets an ErasureCoding policy on a directory at the specified path. + Sets an erasure coding policy on a directory at the specified path. `path`: An directory in HDFS. This is a mandatory parameter. Setting a policy only affects newly created files, and does not affect existing files. - `policyName`: The ErasureCoding policy to be used for files under this directory. + `policyName`: The erasure coding policy to be used for files under this directory. * `[-getPolicy -path ]` - Get details of the ErasureCoding policy of a file or directory at the specified path. + Get details of the erasure coding policy of a file or directory at the specified path. * `[-unsetPolicy -path ]` - Unset an ErasureCoding policy set by a previous call to "setPolicy" on a directory. If the directory inherits the ErasureCoding policy from an ancestor directory, "unsetPolicy" is a no-op. Unsetting the policy on a directory which doesn't have an explicit policy set will not return an error. + Unset an erasure coding policy set by a previous call to `setPolicy` on a directory. If the directory inherits the erasure coding policy from an ancestor directory, `unsetPolicy` is a no-op. Unsetting the policy on a directory which doesn't have an explicit policy set will not return an error. * `[-listPolicies]` - Lists all supported ErasureCoding policies. These names are suitable for use with the `setPolicy` command. + Lists the set of enabled erasure coding policies. These names are suitable for use with the `setPolicy` command. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestErasureCodingCLI.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestErasureCodingCLI.java index 2f4680322a7..718196871e7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestErasureCodingCLI.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestErasureCodingCLI.java @@ -45,6 +45,9 @@ public class TestErasureCodingCLI extends CLITestHelper { public void setUp() throws Exception { super.setUp(); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + "RS-6-3-64k,RS-3-2-64k,XOR-2-1-64k"); + dfsCluster = new MiniDFSCluster.Builder(conf) .numDataNodes(NUM_OF_DATANODES).build(); dfsCluster.waitClusterUp(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java index 60a1a4e4373..7bf5cdcf6ec 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java @@ -68,6 +68,8 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; + import com.google.common.base.Charsets; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; @@ -279,6 +281,15 @@ public class DFSTestUtil { Whitebox.setInternalState(fsn.getFSDirectory(), "editLog", newLog); } + public static void enableAllECPolicies(Configuration conf) { + // Enable all the available EC policies + String policies = + Arrays.asList(ErasureCodingPolicyManager.getSystemPolicies()).stream() + .map(ErasureCodingPolicy::getName) + .collect(Collectors.joining(",")); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, policies); + } + /** class MyFile contains enough information to recreate the contents of * a single file. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedInputStream.java index 3e6d1e40521..fa3e62cbfcc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedInputStream.java @@ -29,7 +29,7 @@ public class TestDFSRSDefault10x4StripedInputStream extends TestDFSStripedInputStream { public ErasureCodingPolicy getEcPolicy() { - return ErasureCodingPolicyManager.getPolicyByPolicyID( + return ErasureCodingPolicyManager.getPolicyByID( HdfsConstants.RS_10_4_POLICY_ID); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStream.java index 1ea839a863b..f4dcdf735da 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStream.java @@ -30,7 +30,7 @@ public class TestDFSRSDefault10x4StripedOutputStream @Override public ErasureCodingPolicy getEcPolicy() { - return ErasureCodingPolicyManager.getPolicyByPolicyID( + return ErasureCodingPolicyManager.getPolicyByID( HdfsConstants.RS_10_4_POLICY_ID); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStreamWithFailure.java index 340fec52fcf..27d3ccfcc3f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStreamWithFailure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStreamWithFailure.java @@ -30,7 +30,7 @@ public class TestDFSRSDefault10x4StripedOutputStreamWithFailure @Override public ErasureCodingPolicy getEcPolicy() { - return ErasureCodingPolicyManager.getPolicyByPolicyID( + return ErasureCodingPolicyManager.getPolicyByID( HdfsConstants.RS_10_4_POLICY_ID); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java index 50b366b26a7..29ef6943af3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java @@ -94,6 +94,8 @@ public class TestDFSStripedInputStream { conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + getEcPolicy().getName()); if (ErasureCodeNative.isNativeCodeLoaded()) { conf.set( CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java index 5956cc12390..675f0422c14 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java @@ -88,6 +88,7 @@ public class TestDFSStripedOutputStream { CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY, NativeRSRawErasureCoderFactory.class.getCanonicalName()); } + DFSTestUtil.enableAllECPolicies(conf); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); cluster.getFileSystem().getClient().setErasureCodingPolicy("/", ecPolicy .getName()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java index 01bd3c569ec..9bd29231fd6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java @@ -217,6 +217,7 @@ public class TestDFSStripedOutputStreamWithFailure { CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY, NativeRSRawErasureCoderFactory.class.getCanonicalName()); } + DFSTestUtil.enableAllECPolicies(conf); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); cluster.waitActive(); dfs = cluster.getFileSystem(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedInputStream.java index 75062e000c0..de0852810af 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedInputStream.java @@ -27,7 +27,7 @@ import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; public class TestDFSXORStripedInputStream extends TestDFSStripedInputStream{ public ErasureCodingPolicy getEcPolicy() { - return ErasureCodingPolicyManager.getPolicyByPolicyID( + return ErasureCodingPolicyManager.getPolicyByID( HdfsConstants.XOR_2_1_POLICY_ID); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedOutputStream.java index 64bddb85607..658c9ba95d7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedOutputStream.java @@ -29,7 +29,7 @@ public class TestDFSXORStripedOutputStream extends TestDFSStripedOutputStream{ @Override public ErasureCodingPolicy getEcPolicy() { - return ErasureCodingPolicyManager.getPolicyByPolicyID( + return ErasureCodingPolicyManager.getPolicyByID( HdfsConstants.XOR_2_1_POLICY_ID); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedOutputStreamWithFailure.java index ed361a8594a..c97644ecdc3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedOutputStreamWithFailure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSXORStripedOutputStreamWithFailure.java @@ -30,7 +30,7 @@ public class TestDFSXORStripedOutputStreamWithFailure @Override public ErasureCodingPolicy getEcPolicy() { - return ErasureCodingPolicyManager.getPolicyByPolicyID( + return ErasureCodingPolicyManager.getPolicyByID( HdfsConstants.XOR_2_1_POLICY_ID); } } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java index e6a822775d1..4a4027b6eaf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java @@ -23,6 +23,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.protocol.DirectoryListing; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; @@ -34,6 +35,7 @@ import org.apache.hadoop.io.erasurecode.ECSchema; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -64,6 +66,7 @@ public class TestErasureCodingPolicies { public void setupCluster() throws IOException { conf = new HdfsConfiguration(); conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE); + DFSTestUtil.enableAllECPolicies(conf); cluster = new MiniDFSCluster.Builder(conf). numDataNodes(1).build(); cluster.waitActive(); @@ -189,6 +192,40 @@ public class TestErasureCodingPolicies { } catch (IOException e) { assertExceptionContains("erasure coding policy for a file", e); } + + // Verify that policies are successfully loaded even when policies + // are disabled + cluster.getConfiguration(0).set( + DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, ""); + cluster.restartNameNodes(); + cluster.waitActive(); + + // No policies should be enabled after restart + Assert.assertTrue("No policies should be enabled after restart", + fs.getAllErasureCodingPolicies().isEmpty()); + + // Already set directory-level policies should still be in effect + Path disabledPolicy = new Path(dir1, "afterDisabled"); + Assert.assertEquals("Dir does not have policy set", + EC_POLICY, + fs.getErasureCodingPolicy(dir1)); + fs.create(disabledPolicy).close(); + Assert.assertEquals("File did not inherit dir's policy", + EC_POLICY, + fs.getErasureCodingPolicy(disabledPolicy)); + + // Also check loading disabled EC policies from fsimage + fs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER); + fs.saveNamespace(); + fs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE); + cluster.restartNameNodes(); + + Assert.assertEquals("Dir does not have policy set", + EC_POLICY, + fs.getErasureCodingPolicy(dir1)); + Assert.assertEquals("File does not have policy set", + EC_POLICY, + fs.getErasureCodingPolicy(disabledPolicy)); } @Test @@ -310,7 +347,7 @@ public class TestErasureCodingPolicies { + "setting an invalid erasure coding policy"); } catch (Exception e) { assertExceptionContains("Policy 'RS-4-2-128k' does not match " + - "any supported erasure coding policies",e); + "any enabled erasure coding policies", e); } } @@ -320,7 +357,7 @@ public class TestErasureCodingPolicies { .getSystemPolicies(); Collection allECPolicies = fs .getAllErasureCodingPolicies(); - assertTrue("All system policies should be active", + assertTrue("All system policies should be enabled", allECPolicies.containsAll(Arrays.asList(sysECPolicies))); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java index 86624cc05b3..ec19a744d54 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java @@ -72,6 +72,7 @@ public class TestUnsetAndChangeDirectoryEcPolicy { CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY, NativeRSRawErasureCoderFactory.class.getCanonicalName()); } + DFSTestUtil.enableAllECPolicies(conf); cluster = new MiniDFSCluster.Builder(conf).numDataNodes( dataBlocks + parityBlocks).build(); cluster.waitActive(); @@ -139,7 +140,7 @@ public class TestUnsetAndChangeDirectoryEcPolicy { final Path ec32FilePath = new Path(childDir, "ec_3_2_file"); final Path ec63FilePath2 = new Path(childDir, "ec_6_3_file_2"); final ErasureCodingPolicy ec32Policy = ErasureCodingPolicyManager - .getPolicyByPolicyID(HdfsConstants.RS_3_2_POLICY_ID); + .getPolicyByID(HdfsConstants.RS_3_2_POLICY_ID); fs.mkdirs(parentDir); fs.setErasureCodingPolicy(parentDir, ecPolicy.getName()); @@ -237,7 +238,7 @@ public class TestUnsetAndChangeDirectoryEcPolicy { final Path ec63FilePath = new Path(rootPath, "ec_6_3_file"); final Path ec32FilePath = new Path(rootPath, "ec_3_2_file"); final ErasureCodingPolicy ec32Policy = ErasureCodingPolicyManager - .getPolicyByPolicyID(HdfsConstants.RS_3_2_POLICY_ID); + .getPolicyByID(HdfsConstants.RS_3_2_POLICY_ID); fs.unsetErasureCodingPolicy(rootPath); fs.setErasureCodingPolicy(rootPath, ecPolicy.getName()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEnabledECPolicies.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEnabledECPolicies.java new file mode 100644 index 00000000000..dd4ae0b81ae --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEnabledECPolicies.java @@ -0,0 +1,151 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode; + +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.apache.hadoop.test.GenericTestUtils; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * Test that ErasureCodingPolicyManager correctly parses the set of enabled + * erasure coding policies from configuration and exposes this information. + */ +public class TestEnabledECPolicies { + + private static final ErasureCodingPolicy[] SYSTEM_POLICIES = + ErasureCodingPolicyManager.getSystemPolicies(); + + @Rule + public Timeout testTimeout = new Timeout(60000); + + private void expectInvalidPolicy(String value) { + HdfsConfiguration conf = new HdfsConfiguration(); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + value); + try { + new ErasureCodingPolicyManager(conf); + fail("Expected exception when instantiating ECPolicyManager"); + } catch (IllegalArgumentException e) { + GenericTestUtils.assertExceptionContains("is not a valid policy", e); + } + } + + private void expectValidPolicy(String value, final int numEnabled) throws + Exception { + HdfsConfiguration conf = new HdfsConfiguration(); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + value); + ErasureCodingPolicyManager manager = new ErasureCodingPolicyManager(conf); + assertEquals("Incorrect number of enabled policies", + numEnabled, manager.getEnabledPolicies().length); + } + + @Test + public void testInvalid() throws Exception { + // Test first with an invalid policy + expectInvalidPolicy("not-a-policy"); + // Test with an invalid policy and a valid policy + expectInvalidPolicy("not-a-policy," + ErasureCodingPolicyManager + .getSystemDefaultPolicy().getName()); + // Test with a valid and an invalid policy + expectInvalidPolicy(ErasureCodingPolicyManager + .getSystemDefaultPolicy().getName() + ", not-a-policy"); + // Some more invalid values + expectInvalidPolicy("not-a-policy, "); + expectInvalidPolicy(" ,not-a-policy, "); + } + + @Test + public void testValid() throws Exception { + String ecPolicyName = ErasureCodingPolicyManager.getSystemDefaultPolicy() + .getName(); + expectValidPolicy(ecPolicyName, 1); + expectValidPolicy(ecPolicyName + ", ", 1); + expectValidPolicy(",", 0); + } + + @Test + public void testGetPolicies() throws Exception { + ErasureCodingPolicy[] enabledPolicies; + // Enable no policies + enabledPolicies = new ErasureCodingPolicy[] + {SYSTEM_POLICIES[1], SYSTEM_POLICIES[2]}; + testGetPolicies(enabledPolicies); + + // Enable one policy + enabledPolicies = new ErasureCodingPolicy[] + {SYSTEM_POLICIES[1]}; + testGetPolicies(enabledPolicies); + + // Enable two policies + enabledPolicies = new ErasureCodingPolicy[] + {SYSTEM_POLICIES[1], SYSTEM_POLICIES[2]}; + testGetPolicies(enabledPolicies); + } + + private void testGetPolicies(ErasureCodingPolicy[] enabledPolicies) + throws Exception { + HdfsConfiguration conf = new HdfsConfiguration(); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + Arrays.asList(enabledPolicies).stream() + .map(ErasureCodingPolicy::getName) + .collect(Collectors.joining(", "))); + ErasureCodingPolicyManager manager = new ErasureCodingPolicyManager(conf); + + // Check that returned values are unique + Set found = new HashSet<>(); + for (ErasureCodingPolicy p : manager.getEnabledPolicies()) { + Assert.assertFalse("Duplicate policy name found: " + p.getName(), + found.contains(p.getName())); + found.add(p.getName()); + } + // Check that the policies specified in conf are found + for (ErasureCodingPolicy p: enabledPolicies) { + Assert.assertTrue("Did not find specified EC policy " + p.getName(), + found.contains(p.getName())); + } + Assert.assertEquals(enabledPolicies.length, found.size()); + // Check that getEnabledPolicyByName only returns enabled policies + for (ErasureCodingPolicy p: SYSTEM_POLICIES) { + if (found.contains(p.getName())) { + // Enabled policy should be present + Assert.assertNotNull( + "getEnabledPolicyByName did not find enabled policy" + p.getName(), + manager.getEnabledPolicyByName(p.getName())); + } else { + // Disabled policy should not be present + Assert.assertNull( + "getEnabledPolicyByName found disabled policy " + p.getName(), + manager.getEnabledPolicyByName(p.getName())); + } + } + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java index 57416ed16d7..e1861643f3f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java @@ -77,7 +77,7 @@ public class TestFSImage { private static final String HADOOP_2_7_ZER0_BLOCK_SIZE_TGZ = "image-with-zero-block-size.tar.gz"; private static final ErasureCodingPolicy testECPolicy = - ErasureCodingPolicyManager.getPolicyByPolicyID( + ErasureCodingPolicyManager.getPolicyByID( HdfsConstants.RS_10_4_POLICY_ID); @Test @@ -240,6 +240,7 @@ public class TestFSImage { @Test public void testSaveAndLoadStripedINodeFile() throws IOException{ Configuration conf = new Configuration(); + DFSTestUtil.enableAllECPolicies(conf); MiniDFSCluster cluster = null; try { cluster = new MiniDFSCluster.Builder(conf).build(); @@ -260,6 +261,7 @@ public class TestFSImage { public void testSaveAndLoadStripedINodeFileUC() throws IOException { // construct a INode with StripedBlock for saving and loading Configuration conf = new Configuration(); + DFSTestUtil.enableAllECPolicies(conf); MiniDFSCluster cluster = null; try { cluster = new MiniDFSCluster.Builder(conf).build(); @@ -459,6 +461,7 @@ public class TestFSImage { final int BLOCK_SIZE = 8 * 1024 * 1024; Configuration conf = new HdfsConfiguration(); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE); + DFSTestUtil.enableAllECPolicies(conf); MiniDFSCluster cluster = null; try { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(GROUP_SIZE) @@ -468,7 +471,7 @@ public class TestFSImage { Path parentDir = new Path("/ec-10-4"); Path childDir = new Path(parentDir, "ec-3-2"); ErasureCodingPolicy ec32Policy = ErasureCodingPolicyManager - .getPolicyByPolicyID(HdfsConstants.RS_3_2_POLICY_ID); + .getPolicyByID(HdfsConstants.RS_3_2_POLICY_ID); // Create directories and files fs.mkdirs(parentDir); @@ -516,7 +519,7 @@ public class TestFSImage { // check the information of file_3_2 inode = fsn.dir.getINode(file_3_2.toString()).asFile(); assertTrue(inode.isStriped()); - assertEquals(ErasureCodingPolicyManager.getPolicyByPolicyID( + assertEquals(ErasureCodingPolicyManager.getPolicyByID( HdfsConstants.RS_3_2_POLICY_ID).getId(), inode.getErasureCodingPolicyID()); blks = inode.getBlocks(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java index e52d5365981..0656249cfca 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java @@ -124,6 +124,10 @@ public class TestOfflineImageViewer { tempDir = Files.createTempDir(); MiniDFSCluster cluster = null; try { + final ErasureCodingPolicy ecPolicy = + ErasureCodingPolicyManager.getPolicyByID( + HdfsConstants.XOR_2_1_POLICY_ID); + Configuration conf = new Configuration(); conf.setLong( DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY, 10000); @@ -134,6 +138,8 @@ public class TestOfflineImageViewer { conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true); conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL, "RULE:[2:$1@$0](JobTracker@.*FOO.COM)s/@.*//" + "DEFAULT"); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + ecPolicy.getName()); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build(); cluster.waitActive(); DistributedFileSystem hdfs = cluster.getFileSystem(); @@ -233,9 +239,6 @@ public class TestOfflineImageViewer { Path ecDir = new Path("/ec"); hdfs.mkdirs(ecDir); dirCount++; - ErasureCodingPolicy ecPolicy = - ErasureCodingPolicyManager.getPolicyByPolicyID( - HdfsConstants.XOR_2_1_POLICY_ID); hdfs.getClient().setErasureCodingPolicy(ecDir.toString(), ecPolicy.getName()); writtenFiles.put(ecDir.toString(), hdfs.getFileStatus(ecDir)); @@ -409,7 +412,7 @@ public class TestOfflineImageViewer { Assert.assertEquals("INode '" + currentInodeName + "' has unexpected EC Policy!", Byte.parseByte(currentECPolicy), - ErasureCodingPolicyManager.getPolicyByPolicyID( + ErasureCodingPolicyManager.getPolicyByID( HdfsConstants.XOR_2_1_POLICY_ID).getId()); Assert.assertEquals("INode '" + currentInodeName + "' has unexpected replication!", diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml index 278963a72d6..8dda6c1af22 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml @@ -119,7 +119,7 @@ SubstringComparator - Get the list of supported erasure coding policies + Get the list of enabled erasure coding policies SubstringComparator @@ -359,7 +359,24 @@ SubstringComparator - Policy 'invalidpolicy' does not match any supported erasure coding policies. + Policy 'invalidpolicy' does not match any enabled erasure coding policies + + + + + + setPolicy : illegal parameters - RS-10-4-64k + + -fs NAMENODE -mkdir /ecdir + -fs NAMENODE -setPolicy -policy RS-10-4-64k -path /ecdir + + + -fs NAMENODE -rmdir /ecdir + + + + SubstringComparator + Policy 'RS-10-4-64k' does not match any enabled erasure coding policies From 570827a819c586b31e88621a9bb1d8118d3c7df3 Mon Sep 17 00:00:00 2001 From: John Zhuge Date: Wed, 8 Mar 2017 23:50:15 -0800 Subject: [PATCH 040/188] HADOOP-14052. Fix dead link in KMS document. Contributed by Christina Vu. Change-Id: I7093f443d93927184196f62f02cc106a2c89e9cf --- hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm b/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm index c1f9b13ba5b..4573b067856 100644 --- a/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm +++ b/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm @@ -956,7 +956,7 @@ $H4 Re-encrypt Encrypted Key With The Latest KeyVersion This command takes a previously generated encrypted key, and re-encrypts it using the latest KeyVersion encryption key in the KeyProvider. If the latest KeyVersion is the same as the one used to generate the encrypted key, the same encrypted key is returned. -This is usually useful after a [Rollover](Rollover_Key) of an encryption key. Re-encrypting the encrypted key will allow it to be encrypted using the latest version of the encryption key, but still with the same key material and initialization vector. +This is usually useful after a [Rollover](#Rollover_Key) of an encryption key. Re-encrypting the encrypted key will allow it to be encrypted using the latest version of the encryption key, but still with the same key material and initialization vector. *REQUEST:* From 385d2cb777a0272ac20c62336c944fad295d5d12 Mon Sep 17 00:00:00 2001 From: Masatake Iwasaki Date: Thu, 9 Mar 2017 13:30:33 +0900 Subject: [PATCH 041/188] HDFS-11499. Decommissioning stuck because of failing recovery. Contributed by Lukas Majercak and Manoj Govindassamy. --- .../server/blockmanagement/BlockManager.java | 10 +++- .../apache/hadoop/hdfs/TestDecommission.java | 48 +++++++++++++++++ .../hadoop/hdfs/TestMaintenanceState.java | 51 +++++++++++++++++++ 3 files changed, 108 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 9ec28f9c02f..5dc40fa10e7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -891,7 +891,15 @@ public class BlockManager implements BlockStatsMXBean { lastBlock.getUnderConstructionFeature() .updateStorageScheduledSize((BlockInfoStriped) lastBlock); } - if (hasMinStorage(lastBlock)) { + + // Count replicas on decommissioning nodes, as these will not be + // decommissioned unless recovery/completing last block has finished + NumberReplicas numReplicas = countNodes(lastBlock); + int numUsableReplicas = numReplicas.liveReplicas() + + numReplicas.decommissioning() + + numReplicas.liveEnteringMaintenanceReplicas(); + + if (hasMinStorage(lastBlock, numUsableReplicas)) { if (committed) { addExpectedReplicasToPending(lastBlock); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java index 94e894619d2..dc0edccdb1a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java @@ -33,6 +33,7 @@ import java.util.concurrent.ExecutionException; import com.google.common.base.Supplier; import com.google.common.collect.Lists; import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -646,6 +647,53 @@ public class TestDecommission extends AdminStatesBaseTest { fdos.close(); } + + @Test(timeout = 360000) + public void testDecommissionWithOpenFileAndBlockRecovery() + throws IOException, InterruptedException { + startCluster(1, 6); + getCluster().waitActive(); + + Path file = new Path("/testRecoveryDecommission"); + + // Create a file and never close the output stream to trigger recovery + DistributedFileSystem dfs = getCluster().getFileSystem(); + FSDataOutputStream out = dfs.create(file, true, + getConf().getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096), + (short) 3, blockSize); + + // Write data to the file + long writtenBytes = 0; + while (writtenBytes < fileSize) { + out.writeLong(writtenBytes); + writtenBytes += 8; + } + out.hsync(); + + DatanodeInfo[] lastBlockLocations = NameNodeAdapter.getBlockLocations( + getCluster().getNameNode(), "/testRecoveryDecommission", 0, fileSize) + .getLastLocatedBlock().getLocations(); + + // Decommission all nodes of the last block + ArrayList toDecom = new ArrayList<>(); + for (DatanodeInfo dnDecom : lastBlockLocations) { + toDecom.add(dnDecom.getXferAddr()); + } + initExcludeHosts(toDecom); + refreshNodes(0); + + // Make sure hard lease expires to trigger replica recovery + getCluster().setLeasePeriod(300L, 300L); + Thread.sleep(2 * BLOCKREPORT_INTERVAL_MSEC); + + for (DatanodeInfo dnDecom : lastBlockLocations) { + DatanodeInfo datanode = NameNodeAdapter.getDatanode( + getCluster().getNamesystem(), dnDecom); + waitNodeState(datanode, AdminStates.DECOMMISSIONED); + } + + assertEquals(dfs.getFileStatus(file).getLen(), writtenBytes); + } /** * Tests restart of namenode while datanode hosts are added to exclude file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMaintenanceState.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMaintenanceState.java index 24321533d13..a37bdb8cb05 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMaintenanceState.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMaintenanceState.java @@ -31,6 +31,8 @@ import java.util.List; import java.util.Map; import com.google.common.collect.Lists; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.junit.Assert; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,8 +43,10 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; import org.apache.hadoop.hdfs.protocol.LocatedBlock; +import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo; +import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.util.Time; import org.junit.Test; @@ -940,6 +944,53 @@ public class TestMaintenanceState extends AdminStatesBaseTest { cleanupFile(fileSys, file); } + @Test(timeout = 120000) + public void testFileCloseAfterEnteringMaintenance() throws Exception { + LOG.info("Starting testFileCloseAfterEnteringMaintenance"); + int expirationInMs = 30 * 1000; + int numDataNodes = 3; + int numNameNodes = 1; + getConf().setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY, 2); + + startCluster(numNameNodes, numDataNodes); + getCluster().waitActive(); + + FSNamesystem fsn = getCluster().getNameNode().getNamesystem(); + List hosts = new ArrayList<>(); + for (DataNode dn : getCluster().getDataNodes()) { + hosts.add(dn.getDisplayName()); + putNodeInService(0, dn.getDatanodeUuid()); + } + assertEquals(numDataNodes, fsn.getNumLiveDataNodes()); + + Path openFile = new Path("/testClosingFileInMaintenance.dat"); + // Lets write 2 blocks of data to the openFile + writeFile(getCluster().getFileSystem(), openFile, (short) 3); + + // Lets write some more data and keep the file open + FSDataOutputStream fsDataOutputStream = getCluster().getFileSystem() + .append(openFile); + byte[] bytes = new byte[1024]; + fsDataOutputStream.write(bytes); + fsDataOutputStream.hsync(); + + LocatedBlocks lbs = NameNodeAdapter.getBlockLocations( + getCluster().getNameNode(0), openFile.toString(), 0, 3 * blockSize); + DatanodeInfo[] dnInfos4LastBlock = lbs.getLastLocatedBlock().getLocations(); + + // Request maintenance for DataNodes 1 and 2 which has the last block. + takeNodeOutofService(0, + Lists.newArrayList(dnInfos4LastBlock[0].getDatanodeUuid(), + dnInfos4LastBlock[1].getDatanodeUuid()), + Time.now() + expirationInMs, + null, null, AdminStates.ENTERING_MAINTENANCE); + + // Closing the file should succeed even when the + // last blocks' nodes are entering maintenance. + fsDataOutputStream.close(); + cleanupFile(getCluster().getFileSystem(), openFile); + } + static String getFirstBlockFirstReplicaUuid(FileSystem fileSys, Path name) throws IOException { DatanodeInfo[] nodes = getFirstBlockReplicasDatanodeInfos(fileSys, name); From 822a74f2ae955ea0893cc02fb36ceb49ceba8014 Mon Sep 17 00:00:00 2001 From: Daniel Templeton Date: Thu, 9 Mar 2017 12:12:47 -0800 Subject: [PATCH 042/188] YARN-6300. NULL_UPDATE_REQUESTS is redundant in TestFairScheduler (Contributed by Yuanbo Liu via Daniel Templeton) --- .../resourcemanager/scheduler/fair/TestFairScheduler.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 31dd7fe5ad6..028eea61829 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -96,7 +96,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdate import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; @@ -119,7 +118,6 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.mockito.Mock; import org.mockito.Mockito; import org.xml.sax.SAXException; @@ -130,8 +128,6 @@ public class TestFairScheduler extends FairSchedulerTestBase { private final int GB = 1024; private final static String ALLOC_FILE = new File(TEST_DIR, "test-queues").getAbsolutePath(); - private final static ContainerUpdates NULL_UPDATE_REQUESTS = - new ContainerUpdates(); @Before public void setUp() throws IOException { From e96a0b8c92b46aed7c1f5ccec13abc6c1043edba Mon Sep 17 00:00:00 2001 From: Sidharta S Date: Thu, 9 Mar 2017 16:22:19 -0800 Subject: [PATCH 043/188] YARN-5669. Add support for docker pull command (Contribtued by luhuichun) --- .../runtime/docker/DockerPullCommand.java | 31 ++++++++++++ .../runtime/docker/TestDockerPullCommand.java | 49 +++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerPullCommand.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerPullCommand.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerPullCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerPullCommand.java new file mode 100644 index 00000000000..351e09ebc00 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerPullCommand.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker; + +/** + * Encapsulates the docker pull command and its command + * line arguments. + */ +public class DockerPullCommand extends DockerCommand { + private static final String PULL_COMMAND = "pull"; + + public DockerPullCommand(String imageName) { + super(PULL_COMMAND); + super.addCommandArguments(imageName); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerPullCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerPullCommand.java new file mode 100644 index 00000000000..89157ff751a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerPullCommand.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker; + +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * Tests the docker pull command and its command + * line arguments. + */ +public class TestDockerPullCommand { + private DockerPullCommand dockerPullCommand; + + private static final String IMAGE_NAME = "foo"; + + @Before + public void setup() { + dockerPullCommand = new DockerPullCommand(IMAGE_NAME); + } + + @Test + public void testGetCommandOption() { + assertEquals("pull", dockerPullCommand.getCommandOption()); + } + + @Test + public void testGetCommandWithArguments() { + assertEquals("pull foo", dockerPullCommand.getCommandWithArguments()); + } + + +} From 819808a016e16325502169e0091a16a6b2ae5387 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Thu, 9 Mar 2017 17:29:11 -0800 Subject: [PATCH 044/188] HDFS-11506. Move ErasureCodingPolicyManager#getSystemDefaultPolicy to test code. Contributed by Manoj Govindassamy. --- .../namenode/ErasureCodingPolicyManager.java | 10 ------ .../org/apache/hadoop/hdfs/DFSTestUtil.java | 2 +- .../hdfs/ErasureCodeBenchmarkThroughput.java | 5 +-- .../hadoop/hdfs/StripedFileTestUtil.java | 12 +++++++ .../hdfs/TestDFSStripedInputStream.java | 3 +- .../hdfs/TestDFSStripedOutputStream.java | 3 +- ...TestDFSStripedOutputStreamWithFailure.java | 3 +- .../hdfs/TestDecommissionWithStriped.java | 5 ++- .../hdfs/TestErasureCodingPolicies.java | 8 ++--- .../TestErasureCodingPolicyWithSnapshot.java | 3 +- .../apache/hadoop/hdfs/TestFileChecksum.java | 5 ++- .../hdfs/TestFileStatusWithECPolicy.java | 3 +- .../hadoop/hdfs/TestLeaseRecoveryStriped.java | 3 +- .../hdfs/TestReadStripedFileWithDecoding.java | 5 ++- .../TestReadStripedFileWithMissingBlocks.java | 3 +- .../hdfs/TestReconstructStripedFile.java | 7 ++-- .../hdfs/TestSafeModeWithStripedFile.java | 5 ++- .../TestUnsetAndChangeDirectoryEcPolicy.java | 3 +- .../hadoop/hdfs/TestWriteReadStripedFile.java | 5 ++- .../hdfs/TestWriteStripedFileWithFailure.java | 5 ++- .../hadoop/hdfs/protocolPB/TestPBHelper.java | 12 +++---- .../hdfs/server/balancer/TestBalancer.java | 5 ++- .../blockmanagement/TestBlockInfoStriped.java | 4 +-- .../TestBlockTokenWithDFSStriped.java | 6 ++-- .../TestLowRedundancyBlockQueues.java | 4 +-- ...nstructStripedBlocksWithRackAwareness.java | 10 +++--- .../TestSequentialBlockGroupId.java | 6 ++-- .../TestSortLocatedStripedBlock.java | 4 +-- .../server/datanode/TestBlockRecovery.java | 3 +- .../TestDataNodeErasureCodingMetrics.java | 5 ++- .../hadoop/hdfs/server/mover/TestMover.java | 5 ++- .../TestAddOverReplicatedStripedBlocks.java | 6 ++-- .../namenode/TestAddStripedBlockInFBR.java | 5 +-- .../server/namenode/TestAddStripedBlocks.java | 7 ++-- .../namenode/TestEnabledECPolicies.java | 12 +++---- .../server/namenode/TestFSEditLogLoader.java | 3 +- .../hadoop/hdfs/server/namenode/TestFsck.java | 33 ++++++++----------- .../server/namenode/TestNameNodeMXBean.java | 12 +++---- .../namenode/TestQuotaWithStripedBlocks.java | 3 +- .../TestReconstructStripedBlocks.java | 6 ++-- .../server/namenode/TestStripedINodeFile.java | 5 +-- ...stOfflineImageViewerWithStripedBlocks.java | 8 ++--- 42 files changed, 121 insertions(+), 141 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java index 02cbbdf38fd..29af2072995 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java @@ -129,16 +129,6 @@ public final class ErasureCodingPolicyManager { return SYS_POLICIES; } - /** - * Get system-wide default policy, which can be used by default - * when no policy is specified for a path. - * @return ecPolicy - */ - public static ErasureCodingPolicy getSystemDefaultPolicy() { - // make this configurable? - return SYS_POLICY1; - } - /** * Get a policy by policy ID. * @return ecPolicy, or null if not found diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java index 7bf5cdcf6ec..13291952a9e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java @@ -1910,7 +1910,7 @@ public class DFSTestUtil { Path dir, int numBlocks, int numStripesPerBlk, boolean toMkdir) throws Exception { createStripedFile(cluster, file, dir, numBlocks, numStripesPerBlk, - toMkdir, ErasureCodingPolicyManager.getSystemDefaultPolicy()); + toMkdir, StripedFileTestUtil.getDefaultECPolicy()); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ErasureCodeBenchmarkThroughput.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ErasureCodeBenchmarkThroughput.java index d1a7569fa02..20ddfd18652 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ErasureCodeBenchmarkThroughput.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ErasureCodeBenchmarkThroughput.java @@ -28,7 +28,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.util.StopWatch; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; @@ -42,9 +41,7 @@ import java.util.Random; import java.util.concurrent.Callable; import java.util.concurrent.CompletionService; import java.util.concurrent.ExecutorCompletionService; -import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; /** @@ -81,7 +78,7 @@ public class ErasureCodeBenchmarkThroughput private static final String EC_FILE_BASE = "ec-file-"; private static final String TMP_FILE_SUFFIX = ".tmp"; private static final ErasureCodingPolicy ecPolicy = - ErasureCodingPolicyManager.getSystemDefaultPolicy(); + StripedFileTestUtil.getDefaultECPolicy(); private static final byte[] data = new byte[BUFFER_SIZE_MB * 1024 * 1024]; static { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java index 520d0e3134a..8008ed3396e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java @@ -29,9 +29,11 @@ import org.apache.hadoop.hdfs.client.impl.BlockReaderTestUtil; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock; +import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.hdfs.util.StripedBlockUtil; import org.apache.hadoop.hdfs.web.WebHdfsFileSystem.WebHdfsInputStream; import org.apache.hadoop.io.IOUtils; @@ -558,4 +560,14 @@ public class StripedFileTestUtil { throws IOException { return fs.getClient().getLocatedBlocks(file.toString(), 0, Long.MAX_VALUE); } + + /** + * Get system-wide default Erasure Coding Policy, which can be + * used by default when no policy is specified for a path. + * @return ErasureCodingPolicy + */ + public static ErasureCodingPolicy getDefaultECPolicy() { + return ErasureCodingPolicyManager.getPolicyByID( + HdfsConstants.RS_6_3_POLICY_ID); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java index 29ef6943af3..68fde95ca8e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java @@ -29,7 +29,6 @@ import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.hdfs.util.StripedBlockUtil; import org.apache.hadoop.io.erasurecode.CodecUtil; import org.apache.hadoop.io.erasurecode.ErasureCodeNative; @@ -76,7 +75,7 @@ public class TestDFSStripedInputStream { public Timeout globalTimeout = new Timeout(300000); public ErasureCodingPolicy getEcPolicy() { - return ErasureCodingPolicyManager.getSystemDefaultPolicy(); + return StripedFileTestUtil.getDefaultECPolicy(); } @Before diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java index 675f0422c14..ebdecfd6406 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java @@ -26,7 +26,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.io.erasurecode.CodecUtil; import org.apache.hadoop.io.erasurecode.ErasureCodeNative; import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory; @@ -62,7 +61,7 @@ public class TestDFSStripedOutputStream { public Timeout globalTimeout = new Timeout(300000); public ErasureCodingPolicy getEcPolicy() { - return ErasureCodingPolicyManager.getSystemDefaultPolicy(); + return StripedFileTestUtil.getDefaultECPolicy(); } @Before diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java index 9bd29231fd6..c66c7f27b71 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java @@ -36,7 +36,6 @@ import org.apache.hadoop.hdfs.security.token.block.SecurityTestUtil; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy; import org.apache.hadoop.hdfs.server.datanode.DataNode; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.io.erasurecode.CodecUtil; import org.apache.hadoop.io.erasurecode.ErasureCodeNative; @@ -89,7 +88,7 @@ public class TestDFSStripedOutputStreamWithFailure { 9 * DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT + 1; public ErasureCodingPolicy getEcPolicy() { - return ErasureCodingPolicyManager.getSystemDefaultPolicy(); + return StripedFileTestUtil.getDefaultECPolicy(); } /* diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java index f8ed0c3f697..a5df85fcb90 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java @@ -47,7 +47,6 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.datanode.DataNode; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; @@ -83,7 +82,7 @@ public class TestDecommissionWithStriped { private MiniDFSCluster cluster; private DistributedFileSystem dfs; private final ErasureCodingPolicy ecPolicy = - ErasureCodingPolicyManager.getSystemDefaultPolicy(); + StripedFileTestUtil.getDefaultECPolicy(); private int numDNs; private final int cellSize = ecPolicy.getCellSize(); private final int dataBlocks = ecPolicy.getNumDataUnits(); @@ -143,7 +142,7 @@ public class TestDecommissionWithStriped { dfs.mkdirs(ecDir); dfs.setErasureCodingPolicy(ecDir, - ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); + StripedFileTestUtil.getDefaultECPolicy().getName()); } @After diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java index 4a4027b6eaf..5ba4403d2ed 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java @@ -56,7 +56,7 @@ public class TestErasureCodingPolicies { private DistributedFileSystem fs; private static final int BLOCK_SIZE = 1024; private static final ErasureCodingPolicy EC_POLICY = - ErasureCodingPolicyManager.getSystemDefaultPolicy(); + StripedFileTestUtil.getDefaultECPolicy(); private FSNamesystem namesystem; @Rule @@ -95,7 +95,7 @@ public class TestErasureCodingPolicies { // set ec policy on dir fs.setErasureCodingPolicy(dir, - ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); + StripedFileTestUtil.getDefaultECPolicy().getName()); // create a file which should be using ec final Path ecSubDir = new Path(dir, "ecSubDir"); final Path ecFile = new Path(ecSubDir, "ecFile"); @@ -270,7 +270,7 @@ public class TestErasureCodingPolicies { final Path testDir = new Path("/ec"); fs.mkdir(testDir, FsPermission.getDirDefault()); fs.setErasureCodingPolicy(testDir, - ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); + StripedFileTestUtil.getDefaultECPolicy().getName()); final Path fooFile = new Path(testDir, "foo"); // create ec file with replication=0 fs.create(fooFile, FsPermission.getFileDefault(), true, @@ -292,7 +292,7 @@ public class TestErasureCodingPolicies { assertNull(fs.getClient().getFileInfo(src).getErasureCodingPolicy()); // dir EC policy after setting ErasureCodingPolicy sysDefaultECPolicy = - ErasureCodingPolicyManager.getSystemDefaultPolicy(); + StripedFileTestUtil.getDefaultECPolicy(); fs.getClient().setErasureCodingPolicy(src, sysDefaultECPolicy.getName()); verifyErasureCodingInfo(src, sysDefaultECPolicy); fs.create(new Path(ecDir, "child1")).close(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java index 6cf4ef42c72..4833c24d2e6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java @@ -28,7 +28,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.util.ToolRunner; import org.junit.After; import org.junit.Before; @@ -41,7 +40,7 @@ public class TestErasureCodingPolicyWithSnapshot { private final static int SUCCESS = 0; private final ErasureCodingPolicy sysDefaultPolicy = - ErasureCodingPolicyManager.getSystemDefaultPolicy(); + StripedFileTestUtil.getDefaultECPolicy(); private final short groupSize = (short) ( sysDefaultPolicy.getNumDataUnits() + sysDefaultPolicy.getNumParityUnits()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java index 8ae176ffef3..7f63e18abd7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java @@ -26,7 +26,6 @@ import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.server.datanode.DataNode; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -47,7 +46,7 @@ public class TestFileChecksum { private static final Logger LOG = LoggerFactory .getLogger(TestFileChecksum.class); private final ErasureCodingPolicy ecPolicy = - ErasureCodingPolicyManager.getSystemDefaultPolicy(); + StripedFileTestUtil.getDefaultECPolicy(); private int dataBlocks = ecPolicy.getNumDataUnits(); private int parityBlocks = ecPolicy.getNumParityUnits(); @@ -82,7 +81,7 @@ public class TestFileChecksum { Path ecPath = new Path(ecDir); cluster.getFileSystem().mkdir(ecPath, FsPermission.getDirDefault()); cluster.getFileSystem().getClient().setErasureCodingPolicy(ecDir, - ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); + StripedFileTestUtil.getDefaultECPolicy().getName()); fs = cluster.getFileSystem(); client = fs.getClient(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java index d7b73274a51..3be0e8d585a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java @@ -26,7 +26,6 @@ import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.junit.After; import org.junit.Before; @@ -71,7 +70,7 @@ public class TestFileStatusWithECPolicy { assertNull(client.getFileInfo(file.toString()).getErasureCodingPolicy()); fs.delete(file, true); - final ErasureCodingPolicy ecPolicy1 = ErasureCodingPolicyManager.getSystemDefaultPolicy(); + final ErasureCodingPolicy ecPolicy1 = StripedFileTestUtil.getDefaultECPolicy(); // set EC policy on dir fs.setErasureCodingPolicy(dir, ecPolicy1.getName()); final ErasureCodingPolicy ecPolicy2 = client.getFileInfo(dir.toUri().getPath()).getErasureCodingPolicy(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java index 710ff622b40..2ba5aede855 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java @@ -29,7 +29,6 @@ import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.server.datanode.DataNode; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.hdfs.util.StripedBlockUtil; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.security.UserGroupInformation; @@ -57,7 +56,7 @@ public class TestLeaseRecoveryStriped { .getLog(TestLeaseRecoveryStriped.class); private final ErasureCodingPolicy ecPolicy = - ErasureCodingPolicyManager.getSystemDefaultPolicy(); + StripedFileTestUtil.getDefaultECPolicy(); private final int dataBlocks = ecPolicy.getNumDataUnits(); private final int parityBlocks = ecPolicy.getNumParityUnits(); private final int cellSize = ecPolicy.getCellSize(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java index d7d9cc1c20c..4c6ad99c885 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java @@ -35,7 +35,6 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; @@ -68,7 +67,7 @@ public class TestReadStripedFileWithDecoding { private MiniDFSCluster cluster; private DistributedFileSystem fs; private final ErasureCodingPolicy ecPolicy = - ErasureCodingPolicyManager.getSystemDefaultPolicy(); + StripedFileTestUtil.getDefaultECPolicy(); private final short dataBlocks = (short) ecPolicy.getNumDataUnits(); private final short parityBlocks = (short) ecPolicy.getNumParityUnits(); @@ -103,7 +102,7 @@ public class TestReadStripedFileWithDecoding { false); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); cluster.getFileSystem().getClient().setErasureCodingPolicy("/", - ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); + StripedFileTestUtil.getDefaultECPolicy().getName()); fs = cluster.getFileSystem(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java index b65626e149c..930986f0764 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java @@ -24,7 +24,6 @@ import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.server.datanode.DataNode; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.junit.Assert; import org.junit.Test; import org.junit.Rule; @@ -43,7 +42,7 @@ public class TestReadStripedFileWithMissingBlocks { private DistributedFileSystem fs; private Configuration conf = new HdfsConfiguration(); private final ErasureCodingPolicy ecPolicy = - ErasureCodingPolicyManager.getSystemDefaultPolicy(); + StripedFileTestUtil.getDefaultECPolicy(); private final short dataBlocks = (short) ecPolicy.getNumDataUnits(); private final short parityBlocks = (short) ecPolicy.getNumParityUnits(); private final int cellSize = ecPolicy.getCellSize(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java index 38939f5482e..88edcb40d82 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java @@ -44,7 +44,6 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo; import org.apache.hadoop.hdfs.server.datanode.DataNode; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.hdfs.server.protocol.BlockECReconstructionCommand.BlockECReconstructionInfo; import org.apache.hadoop.hdfs.util.StripedBlockUtil; @@ -62,7 +61,7 @@ public class TestReconstructStripedFile { public static final Log LOG = LogFactory.getLog(TestReconstructStripedFile.class); private final ErasureCodingPolicy ecPolicy = - ErasureCodingPolicyManager.getSystemDefaultPolicy(); + StripedFileTestUtil.getDefaultECPolicy(); private final int dataBlkNum = ecPolicy.getNumDataUnits(); private final int parityBlkNum = ecPolicy.getNumParityUnits(); private final int cellSize = ecPolicy.getCellSize(); @@ -108,7 +107,7 @@ public class TestReconstructStripedFile { fs = cluster.getFileSystem(); fs.getClient().setErasureCodingPolicy("/", - ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); + StripedFileTestUtil.getDefaultECPolicy().getName()); List datanodes = cluster.getDataNodes(); for (int i = 0; i < dnNum; i++) { @@ -418,7 +417,7 @@ public class TestReconstructStripedFile { BlockECReconstructionInfo invalidECInfo = new BlockECReconstructionInfo( new ExtendedBlock("bp-id", 123456), dataDNs, dnStorageInfo, liveIndices, - ErasureCodingPolicyManager.getSystemDefaultPolicy()); + StripedFileTestUtil.getDefaultECPolicy()); List ecTasks = new ArrayList<>(); ecTasks.add(invalidECInfo); dataNode.getErasureCodingWorker().processErasureCodingTasks(ecTasks); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java index 0731779681f..85538ba9161 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java @@ -25,7 +25,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.junit.After; @@ -44,7 +43,7 @@ import static org.junit.Assert.assertTrue; public class TestSafeModeWithStripedFile { private final ErasureCodingPolicy ecPolicy = - ErasureCodingPolicyManager.getSystemDefaultPolicy(); + StripedFileTestUtil.getDefaultECPolicy(); private final short dataBlocks = (short) ecPolicy.getNumDataUnits(); private final short parityBlocks = (short) ecPolicy.getNumParityUnits(); private final int numDNs = dataBlocks + parityBlocks; @@ -64,7 +63,7 @@ public class TestSafeModeWithStripedFile { conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 100); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); cluster.getFileSystem().getClient().setErasureCodingPolicy("/", - ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); + StripedFileTestUtil.getDefaultECPolicy().getName()); cluster.waitActive(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java index ec19a744d54..32f6bc8013c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java @@ -51,8 +51,7 @@ public class TestUnsetAndChangeDirectoryEcPolicy { private MiniDFSCluster cluster; private Configuration conf = new Configuration(); private DistributedFileSystem fs; - private ErasureCodingPolicy ecPolicy = ErasureCodingPolicyManager - .getSystemDefaultPolicy(); + private ErasureCodingPolicy ecPolicy = StripedFileTestUtil.getDefaultECPolicy(); private final short dataBlocks = (short) ecPolicy.getNumDataUnits(); private final short parityBlocks = (short) ecPolicy.getNumParityUnits(); private final int cellSize = ecPolicy.getCellSize(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java index 76ca704e934..9aac65b411e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java @@ -27,7 +27,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy; import org.apache.hadoop.hdfs.server.datanode.DataNode; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.hdfs.web.WebHdfsConstants; import org.apache.hadoop.hdfs.web.WebHdfsTestUtil; import org.apache.hadoop.ipc.RemoteException; @@ -48,7 +47,7 @@ import java.util.Random; public class TestWriteReadStripedFile { public static final Log LOG = LogFactory.getLog(TestWriteReadStripedFile.class); private final ErasureCodingPolicy ecPolicy = - ErasureCodingPolicyManager.getSystemDefaultPolicy(); + StripedFileTestUtil.getDefaultECPolicy(); private final int cellSize = ecPolicy.getCellSize(); private final short dataBlocks = (short) ecPolicy.getNumDataUnits(); private final short parityBlocks = (short) ecPolicy.getNumParityUnits(); @@ -81,7 +80,7 @@ public class TestWriteReadStripedFile { fs = cluster.getFileSystem(); fs.mkdirs(new Path("/ec")); cluster.getFileSystem().getClient().setErasureCodingPolicy("/ec", - ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); + StripedFileTestUtil.getDefaultECPolicy().getName()); } @After diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java index 23a98214900..03e9e10bf13 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java @@ -24,7 +24,6 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.test.GenericTestUtils; import org.apache.log4j.Level; import org.junit.Assert; @@ -47,7 +46,7 @@ public class TestWriteStripedFileWithFailure { } private final ErasureCodingPolicy ecPolicy = - ErasureCodingPolicyManager.getSystemDefaultPolicy(); + StripedFileTestUtil.getDefaultECPolicy(); private final short dataBlocks = (short) ecPolicy.getNumDataUnits(); private final short parityBlocks = (short) ecPolicy.getNumParityUnits(); private final int numDNs = dataBlocks + parityBlocks; @@ -60,7 +59,7 @@ public class TestWriteStripedFileWithFailure { conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); cluster.getFileSystem().getClient().setErasureCodingPolicy("/", - ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); + StripedFileTestUtil.getDefaultECPolicy().getName()); fs = cluster.getFileSystem(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java index ff0852809ad..ff4b8ec8b1c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java @@ -36,6 +36,7 @@ import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockType; import org.apache.hadoop.hdfs.protocol.DatanodeID; @@ -77,7 +78,6 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType; import org.apache.hadoop.hdfs.server.common.StorageInfo; import org.apache.hadoop.hdfs.server.namenode.CheckpointSignature; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.hdfs.server.protocol.BlockCommand; import org.apache.hadoop.hdfs.server.protocol.BlockECReconstructionCommand.BlockECReconstructionInfo; import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand; @@ -228,7 +228,7 @@ public class TestPBHelper { datanodeUuids, storageIDs, storageTypes); if (isStriped) { blkLocs = new StripedBlockWithLocations(blkLocs, indices, dataBlkNum, - ErasureCodingPolicyManager.getSystemDefaultPolicy().getCellSize()); + StripedFileTestUtil.getDefaultECPolicy().getCellSize()); } return blkLocs; } @@ -720,7 +720,7 @@ public class TestPBHelper { byte[] liveBlkIndices0 = new byte[2]; BlockECReconstructionInfo blkECRecoveryInfo0 = new BlockECReconstructionInfo( new ExtendedBlock("bp1", 1234), dnInfos0, targetDnInfos0, - liveBlkIndices0, ErasureCodingPolicyManager.getSystemDefaultPolicy()); + liveBlkIndices0, StripedFileTestUtil.getDefaultECPolicy()); DatanodeInfo[] dnInfos1 = new DatanodeInfo[] { DFSTestUtil.getLocalDatanodeInfo(), DFSTestUtil.getLocalDatanodeInfo() }; DatanodeStorageInfo targetDnInfos_2 = BlockManagerTestUtil @@ -734,7 +734,7 @@ public class TestPBHelper { byte[] liveBlkIndices1 = new byte[2]; BlockECReconstructionInfo blkECRecoveryInfo1 = new BlockECReconstructionInfo( new ExtendedBlock("bp2", 3256), dnInfos1, targetDnInfos1, - liveBlkIndices1, ErasureCodingPolicyManager.getSystemDefaultPolicy()); + liveBlkIndices1, StripedFileTestUtil.getDefaultECPolicy()); List blkRecoveryInfosList = new ArrayList(); blkRecoveryInfosList.add(blkECRecoveryInfo0); blkRecoveryInfosList.add(blkECRecoveryInfo1); @@ -823,8 +823,8 @@ public class TestPBHelper { ErasureCodingPolicy ecPolicy2 = blkECRecoveryInfo2.getErasureCodingPolicy(); // Compare ECPolicies same as default ECPolicy as we used system default // ECPolicy used in this test - compareECPolicies(ErasureCodingPolicyManager.getSystemDefaultPolicy(), ecPolicy1); - compareECPolicies(ErasureCodingPolicyManager.getSystemDefaultPolicy(), ecPolicy2); + compareECPolicies(StripedFileTestUtil.getDefaultECPolicy(), ecPolicy1); + compareECPolicies(StripedFileTestUtil.getDefaultECPolicy(), ecPolicy2); } private void compareECPolicies(ErasureCodingPolicy ecPolicy1, ErasureCodingPolicy ecPolicy2) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java index 70aa4e08ca5..d9a8e5b1659 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java @@ -46,7 +46,6 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBER import static org.apache.hadoop.test.PlatformAssumptions.assumeNotWindows; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.junit.AfterClass; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -203,7 +202,7 @@ public class TestBalancer { } private final ErasureCodingPolicy ecPolicy = - ErasureCodingPolicyManager.getSystemDefaultPolicy(); + StripedFileTestUtil.getDefaultECPolicy(); private final int dataBlocks = ecPolicy.getNumDataUnits(); private final int parityBlocks = ecPolicy.getNumParityUnits(); private final int groupSize = dataBlocks + parityBlocks; @@ -1941,7 +1940,7 @@ public class TestBalancer { client = NameNodeProxies.createProxy(conf, cluster.getFileSystem(0).getUri(), ClientProtocol.class).getProxy(); client.setErasureCodingPolicy("/", - ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); + StripedFileTestUtil.getDefaultECPolicy().getName()); long totalCapacity = sum(capacities); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java index 6a55d8bf336..1040d218578 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java @@ -18,8 +18,8 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.protocol.Block; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.junit.Assert; import org.junit.Rule; @@ -43,7 +43,7 @@ public class TestBlockInfoStriped { private static final long BASE_ID = -1600; private final Block baseBlock = new Block(BASE_ID); private final ErasureCodingPolicy testECPolicy - = ErasureCodingPolicyManager.getSystemDefaultPolicy(); + = StripedFileTestUtil.getDefaultECPolicy(); private final int totalBlocks = testECPolicy.getNumDataUnits() + testECPolicy.getNumParityUnits(); private final BlockInfoStriped info = new BlockInfoStriped(baseBlock, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java index 20c3accbd3e..82c9fde1d5a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java @@ -20,11 +20,11 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock; import org.apache.hadoop.hdfs.server.balancer.TestBalancer; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.hdfs.util.StripedBlockUtil; import org.apache.hadoop.net.ServerSocketUtil; import org.junit.Rule; @@ -35,7 +35,7 @@ import java.io.IOException; public class TestBlockTokenWithDFSStriped extends TestBlockTokenWithDFS { private final ErasureCodingPolicy ecPolicy = - ErasureCodingPolicyManager.getSystemDefaultPolicy(); + StripedFileTestUtil.getDefaultECPolicy(); private final int dataBlocks = ecPolicy.getNumDataUnits(); private final int parityBlocks = ecPolicy.getNumParityUnits(); private final int cellSize = ecPolicy.getCellSize(); @@ -84,7 +84,7 @@ public class TestBlockTokenWithDFSStriped extends TestBlockTokenWithDFS { .numDataNodes(numDNs) .build(); cluster.getFileSystem().getClient().setErasureCodingPolicy("/", - ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); + StripedFileTestUtil.getDefaultECPolicy().getName()); try { cluster.waitActive(); doTestRead(conf, cluster, true); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestLowRedundancyBlockQueues.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestLowRedundancyBlockQueues.java index 2eb7abff88c..d853762a6e9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestLowRedundancyBlockQueues.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestLowRedundancyBlockQueues.java @@ -20,8 +20,8 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import java.util.Iterator; +import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.protocol.Block; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.junit.Test; @@ -33,7 +33,7 @@ import static org.junit.Assert.fail; public class TestLowRedundancyBlockQueues { private final ErasureCodingPolicy ecPolicy = - ErasureCodingPolicyManager.getSystemDefaultPolicy(); + StripedFileTestUtil.getDefaultECPolicy(); private BlockInfo genBlockInfo(long id) { return new BlockInfoContiguous(new Block(id), (short) 3); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java index 832876e4492..9b7c7cd81ed 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java @@ -23,13 +23,13 @@ import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.INodeFile; import org.apache.hadoop.net.NetworkTopology; @@ -59,7 +59,7 @@ public class TestReconstructStripedBlocksWithRackAwareness { } private final ErasureCodingPolicy ecPolicy = - ErasureCodingPolicyManager.getSystemDefaultPolicy(); + StripedFileTestUtil.getDefaultECPolicy(); private final int cellSize = ecPolicy.getCellSize(); private final short dataBlocks = (short) ecPolicy.getNumDataUnits(); private final short parityBlocks = (short) ecPolicy.getNumParityUnits(); @@ -151,7 +151,7 @@ public class TestReconstructStripedBlocksWithRackAwareness { cluster.waitActive(); fs = cluster.getFileSystem(); fs.setErasureCodingPolicy(new Path("/"), - ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); + StripedFileTestUtil.getDefaultECPolicy().getName()); FSNamesystem fsn = cluster.getNamesystem(); BlockManager bm = fsn.getBlockManager(); @@ -222,7 +222,7 @@ public class TestReconstructStripedBlocksWithRackAwareness { cluster.waitActive(); fs = cluster.getFileSystem(); fs.setErasureCodingPolicy(new Path("/"), - ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); + StripedFileTestUtil.getDefaultECPolicy().getName()); MiniDFSCluster.DataNodeProperties lastHost = stopDataNode( hosts[hosts.length - 1]); @@ -276,7 +276,7 @@ public class TestReconstructStripedBlocksWithRackAwareness { cluster.waitActive(); fs = cluster.getFileSystem(); fs.setErasureCodingPolicy(new Path("/"), - ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); + StripedFileTestUtil.getDefaultECPolicy().getName()); final BlockManager bm = cluster.getNamesystem().getBlockManager(); final DatanodeManager dm = bm.getDatanodeManager(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java index 7920c593075..3c18112b566 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java @@ -37,9 +37,9 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.LocatedBlock; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; @@ -58,7 +58,7 @@ public class TestSequentialBlockGroupId { .getLog("TestSequentialBlockGroupId"); private final ErasureCodingPolicy ecPolicy = - ErasureCodingPolicyManager.getSystemDefaultPolicy(); + StripedFileTestUtil.getDefaultECPolicy(); private final short REPLICATION = 1; private final long SEED = 0; private final int dataBlocks = ecPolicy.getNumDataUnits(); @@ -89,7 +89,7 @@ public class TestSequentialBlockGroupId { .getBlockIdManager().getBlockGroupIdGenerator(); fs.mkdirs(ecDir); cluster.getFileSystem().getClient().setErasureCodingPolicy("/ecDir", - ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); + StripedFileTestUtil.getDefaultECPolicy().getName()); } @After diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSortLocatedStripedBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSortLocatedStripedBlock.java index 4db361740bd..616b4c340dd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSortLocatedStripedBlock.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSortLocatedStripedBlock.java @@ -27,6 +27,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; @@ -34,7 +35,6 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock; import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.Time; @@ -54,7 +54,7 @@ public class TestSortLocatedStripedBlock { .getLogger(TestSortLocatedStripedBlock.class); private final ErasureCodingPolicy ecPolicy = - ErasureCodingPolicyManager.getSystemDefaultPolicy(); + StripedFileTestUtil.getDefaultECPolicy(); private final int cellSize = ecPolicy.getCellSize(); private final short dataBlocks = (short) ecPolicy.getNumDataUnits(); private final short parityBlocks = (short) ecPolicy.getNumParityUnits(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java index b64f1e29f33..9f0011c8a5e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java @@ -65,6 +65,7 @@ import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.server.protocol.SlowPeerReports; import org.apache.hadoop.util.AutoCloseableLock; import org.apache.hadoop.hdfs.protocol.DatanodeID; @@ -137,7 +138,7 @@ public class TestBlockRecovery { public TestName currentTestName = new TestName(); private final int cellSize = - ErasureCodingPolicyManager.getSystemDefaultPolicy().getCellSize(); + StripedFileTestUtil.getDefaultECPolicy().getCellSize(); private final int bytesPerChecksum = 512; private final int[][][] blockLengthsSuite = { {{11 * cellSize, 10 * cellSize, 9 * cellSize, 8 * cellSize, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java index 7036c7a45b7..d36db233f5f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java @@ -33,7 +33,6 @@ import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import static org.apache.hadoop.test.MetricsAsserts.getLongCounter; @@ -55,7 +54,7 @@ public class TestDataNodeErasureCodingMetrics { public static final Log LOG = LogFactory. getLog(TestDataNodeErasureCodingMetrics.class); private final ErasureCodingPolicy ecPolicy = - ErasureCodingPolicyManager.getSystemDefaultPolicy(); + StripedFileTestUtil.getDefaultECPolicy(); private final int dataBlocks = ecPolicy.getNumDataUnits(); private final int parityBlocks = ecPolicy.getNumParityUnits(); private final int cellSize = ecPolicy.getCellSize(); @@ -76,7 +75,7 @@ public class TestDataNodeErasureCodingMetrics { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); cluster.waitActive(); cluster.getFileSystem().getClient().setErasureCodingPolicy("/", - ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); + StripedFileTestUtil.getDefaultECPolicy().getName()); fs = cluster.getFileSystem(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java index a403ff4bb05..0fcb2755955 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java @@ -78,7 +78,6 @@ import org.apache.hadoop.hdfs.server.balancer.TestBalancer; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; import org.apache.hadoop.hdfs.server.mover.Mover.MLocation; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil; import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.minikdc.MiniKdc; @@ -478,7 +477,7 @@ public class TestMover { } private final ErasureCodingPolicy ecPolicy = - ErasureCodingPolicyManager.getSystemDefaultPolicy(); + StripedFileTestUtil.getDefaultECPolicy(); private final int dataBlocks = ecPolicy.getNumDataUnits(); private final int parityBlocks = ecPolicy.getNumParityUnits(); private final int cellSize = ecPolicy.getCellSize(); @@ -538,7 +537,7 @@ public class TestMover { HdfsConstants.HOT_STORAGE_POLICY_NAME); // set an EC policy on "/bar" directory client.setErasureCodingPolicy(barDir, - ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); + StripedFileTestUtil.getDefaultECPolicy().getName()); // write file to barDir final String fooFile = "/bar/foo"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java index 670efd69552..6a4cd32a582 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java @@ -56,7 +56,7 @@ public class TestAddOverReplicatedStripedBlocks { private final Path dirPath = new Path("/striped"); private Path filePath = new Path(dirPath, "file"); private final ErasureCodingPolicy ecPolicy = - ErasureCodingPolicyManager.getSystemDefaultPolicy(); + StripedFileTestUtil.getDefaultECPolicy(); private final short dataBlocks = (short) ecPolicy.getNumDataUnits(); private final short parityBlocks = (short) ecPolicy.getNumParityUnits(); private final short groupSize = (short) (dataBlocks + parityBlocks); @@ -82,7 +82,7 @@ public class TestAddOverReplicatedStripedBlocks { fs = cluster.getFileSystem(); fs.mkdirs(dirPath); fs.getClient().setErasureCodingPolicy(dirPath.toString(), - ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); + StripedFileTestUtil.getDefaultECPolicy().getName()); } @After @@ -192,7 +192,7 @@ public class TestAddOverReplicatedStripedBlocks { long groupId = bg.getBlock().getBlockId(); Block blk = new Block(groupId, blockSize, gs); BlockInfoStriped blockInfo = new BlockInfoStriped(blk, - ErasureCodingPolicyManager.getSystemDefaultPolicy()); + StripedFileTestUtil.getDefaultECPolicy()); for (int i = 0; i < groupSize; i++) { blk.setBlockId(groupId + i); cluster.injectBlocks(i, Arrays.asList(blk), bpid); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlockInFBR.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlockInFBR.java index 87fbcc63c54..a3afa93e3aa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlockInFBR.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlockInFBR.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped; @@ -45,7 +46,7 @@ import java.io.IOException; public class TestAddStripedBlockInFBR { private final ErasureCodingPolicy ecPolicy = - ErasureCodingPolicyManager.getSystemDefaultPolicy(); + StripedFileTestUtil.getDefaultECPolicy(); private final int cellSize = ecPolicy.getCellSize(); private final short dataBlocks = (short) ecPolicy.getNumDataUnits(); private final short parityBlocks = (short) ecPolicy.getNumParityUnits(); @@ -88,7 +89,7 @@ public class TestAddStripedBlockInFBR { dfs.mkdirs(ecDir); dfs.mkdirs(repDir); dfs.getClient().setErasureCodingPolicy(ecDir.toString(), - ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); + StripedFileTestUtil.getDefaultECPolicy().getName()); // create several non-EC files and one EC file final Path[] repFiles = new Path[groupSize]; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java index 2df1aa49855..2eb23b9f829 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; import org.apache.hadoop.hdfs.protocol.DatanodeID; @@ -67,7 +68,7 @@ import static org.junit.Assert.assertEquals; public class TestAddStripedBlocks { private final ErasureCodingPolicy ecPolicy = - ErasureCodingPolicyManager.getSystemDefaultPolicy(); + StripedFileTestUtil.getDefaultECPolicy(); private final short dataBlocks = (short) ecPolicy.getNumDataUnits(); private final short parityBlocks = (short) ecPolicy.getNumParityUnits(); private final int cellSize = ecPolicy.getCellSize(); @@ -86,8 +87,8 @@ public class TestAddStripedBlocks { .numDataNodes(groupSize).build(); cluster.waitActive(); dfs = cluster.getFileSystem(); - dfs.getClient().setErasureCodingPolicy("/", ErasureCodingPolicyManager - .getSystemDefaultPolicy().getName()); + dfs.getClient().setErasureCodingPolicy("/", + StripedFileTestUtil.getDefaultECPolicy().getName()); } @After diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEnabledECPolicies.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEnabledECPolicies.java index dd4ae0b81ae..7f18d18d731 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEnabledECPolicies.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEnabledECPolicies.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.namenode; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.test.GenericTestUtils; import org.junit.Assert; @@ -73,11 +74,11 @@ public class TestEnabledECPolicies { // Test first with an invalid policy expectInvalidPolicy("not-a-policy"); // Test with an invalid policy and a valid policy - expectInvalidPolicy("not-a-policy," + ErasureCodingPolicyManager - .getSystemDefaultPolicy().getName()); + expectInvalidPolicy("not-a-policy," + + StripedFileTestUtil.getDefaultECPolicy().getName()); // Test with a valid and an invalid policy - expectInvalidPolicy(ErasureCodingPolicyManager - .getSystemDefaultPolicy().getName() + ", not-a-policy"); + expectInvalidPolicy( + StripedFileTestUtil.getDefaultECPolicy().getName() + ", not-a-policy"); // Some more invalid values expectInvalidPolicy("not-a-policy, "); expectInvalidPolicy(" ,not-a-policy, "); @@ -85,8 +86,7 @@ public class TestEnabledECPolicies { @Test public void testValid() throws Exception { - String ecPolicyName = ErasureCodingPolicyManager.getSystemDefaultPolicy() - .getName(); + String ecPolicyName = StripedFileTestUtil.getDefaultECPolicy().getName(); expectValidPolicy(ecPolicyName, 1); expectValidPolicy(ecPolicyName + ", ", 1); expectValidPolicy(",", 0); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java index 72d76b77ef6..104727d83a2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java @@ -46,6 +46,7 @@ import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; @@ -99,7 +100,7 @@ public class TestFSEditLogLoader { private static final int NUM_DATA_NODES = 0; private final ErasureCodingPolicy testECPolicy - = ErasureCodingPolicyManager.getSystemDefaultPolicy(); + = StripedFileTestUtil.getDefaultECPolicy(); @Test public void testDisplayRecentEditLogOpCodes() throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java index 76c5378062e..b94e6108c4e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java @@ -676,7 +676,7 @@ public class TestFsck { setNumFiles(4).build(); conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L); ErasureCodingPolicy ecPolicy = - ErasureCodingPolicyManager.getSystemDefaultPolicy(); + StripedFileTestUtil.getDefaultECPolicy(); final int dataBlocks = ecPolicy.getNumDataUnits(); final int cellSize = ecPolicy.getCellSize(); final int numAllUnits = dataBlocks + ecPolicy.getNumParityUnits(); @@ -1997,10 +1997,9 @@ public class TestFsck { conf.setLong(DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY, precision); conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L); - int dataBlocks = ErasureCodingPolicyManager - .getSystemDefaultPolicy().getNumDataUnits(); - int parityBlocks = ErasureCodingPolicyManager - .getSystemDefaultPolicy().getNumParityUnits(); + int dataBlocks = StripedFileTestUtil.getDefaultECPolicy().getNumDataUnits(); + int parityBlocks = + StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits(); int totalSize = dataBlocks + parityBlocks; cluster = new MiniDFSCluster.Builder(conf).numDataNodes(totalSize).build(); fs = cluster.getFileSystem(); @@ -2288,12 +2287,10 @@ public class TestFsck { @Test (timeout = 300000) public void testFsckCorruptECFile() throws Exception { DistributedFileSystem fs = null; - int dataBlocks = ErasureCodingPolicyManager - .getSystemDefaultPolicy().getNumDataUnits(); - int parityBlocks = ErasureCodingPolicyManager - .getSystemDefaultPolicy().getNumParityUnits(); - int cellSize = ErasureCodingPolicyManager - .getSystemDefaultPolicy().getCellSize(); + int dataBlocks = StripedFileTestUtil.getDefaultECPolicy().getNumDataUnits(); + int parityBlocks = + StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits(); + int cellSize = StripedFileTestUtil.getDefaultECPolicy().getCellSize(); int totalSize = dataBlocks + parityBlocks; cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(totalSize).build(); @@ -2308,7 +2305,7 @@ public class TestFsck { Path ecDirPath = new Path("/striped"); fs.mkdir(ecDirPath, FsPermission.getDirDefault()); fs.getClient().setErasureCodingPolicy(ecDirPath.toString(), - ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); + StripedFileTestUtil.getDefaultECPolicy().getName()); Path file = new Path(ecDirPath, "corrupted"); final int length = cellSize * dataBlocks; final byte[] bytes = StripedFileTestUtil.generateBytes(length); @@ -2359,12 +2356,10 @@ public class TestFsck { @Test (timeout = 300000) public void testFsckMissingECFile() throws Exception { DistributedFileSystem fs = null; - int dataBlocks = ErasureCodingPolicyManager - .getSystemDefaultPolicy().getNumDataUnits(); - int parityBlocks = ErasureCodingPolicyManager - .getSystemDefaultPolicy().getNumParityUnits(); - int cellSize = ErasureCodingPolicyManager - .getSystemDefaultPolicy().getCellSize(); + int dataBlocks = StripedFileTestUtil.getDefaultECPolicy().getNumDataUnits(); + int parityBlocks = + StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits(); + int cellSize = StripedFileTestUtil.getDefaultECPolicy().getCellSize(); int totalSize = dataBlocks + parityBlocks; cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(totalSize).build(); @@ -2374,7 +2369,7 @@ public class TestFsck { Path ecDirPath = new Path("/striped"); fs.mkdir(ecDirPath, FsPermission.getDirDefault()); fs.getClient().setErasureCodingPolicy(ecDirPath.toString(), - ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); + StripedFileTestUtil.getDefaultECPolicy().getName()); Path file = new Path(ecDirPath, "missing"); final int length = cellSize * dataBlocks; final byte[] bytes = StripedFileTestUtil.generateBytes(length); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java index d4d47637310..231f75fc904 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java @@ -725,12 +725,10 @@ public class TestNameNodeMXBean { DistributedFileSystem fs = null; try { Configuration conf = new HdfsConfiguration(); - int dataBlocks = ErasureCodingPolicyManager - .getSystemDefaultPolicy().getNumDataUnits(); - int parityBlocks = ErasureCodingPolicyManager - .getSystemDefaultPolicy().getNumParityUnits(); - int cellSize = ErasureCodingPolicyManager - .getSystemDefaultPolicy().getCellSize(); + int dataBlocks = StripedFileTestUtil.getDefaultECPolicy().getNumDataUnits(); + int parityBlocks = + StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits(); + int cellSize = StripedFileTestUtil.getDefaultECPolicy().getCellSize(); int totalSize = dataBlocks + parityBlocks; cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(totalSize).build(); @@ -740,7 +738,7 @@ public class TestNameNodeMXBean { Path ecDirPath = new Path("/striped"); fs.mkdir(ecDirPath, FsPermission.getDirDefault()); fs.getClient().setErasureCodingPolicy(ecDirPath.toString(), - ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); + StripedFileTestUtil.getDefaultECPolicy().getName()); Path file = new Path(ecDirPath, "corrupted"); final int length = cellSize * dataBlocks; final byte[] bytes = StripedFileTestUtil.generateBytes(length); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java index 326ddc84c65..1e836dc09bf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; @@ -46,7 +47,7 @@ public class TestQuotaWithStripedBlocks { private static final int BLOCK_SIZE = 1024 * 1024; private static final long DISK_QUOTA = BLOCK_SIZE * 10; private final ErasureCodingPolicy ecPolicy = - ErasureCodingPolicyManager.getSystemDefaultPolicy(); + StripedFileTestUtil.getDefaultECPolicy(); private final int dataBlocks = ecPolicy.getNumDataUnits(); private final int parityBlocsk = ecPolicy.getNumParityUnits(); private final int groupSize = dataBlocks + parityBlocsk; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java index 5e4a9db39dd..01a8e08bb1a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java @@ -60,7 +60,7 @@ public class TestReconstructStripedBlocks { public static final Logger LOG = LoggerFactory.getLogger( TestReconstructStripedBlocks.class); private final ErasureCodingPolicy ecPolicy = - ErasureCodingPolicyManager.getSystemDefaultPolicy(); + StripedFileTestUtil.getDefaultECPolicy(); private final int cellSize = ecPolicy.getCellSize(); private final short dataBlocks = (short) ecPolicy.getNumDataUnits(); private final short parityBlocks = (short) ecPolicy.getNumParityUnits(); @@ -202,7 +202,7 @@ public class TestReconstructStripedBlocks { DistributedFileSystem fs = cluster.getFileSystem(); BlockManager bm = cluster.getNamesystem().getBlockManager(); fs.getClient().setErasureCodingPolicy("/", - ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); + StripedFileTestUtil.getDefaultECPolicy().getName()); int fileLen = dataBlocks * blockSize; Path p = new Path("/test2RecoveryTasksForSameBlockGroup"); final byte[] data = new byte[fileLen]; @@ -268,7 +268,7 @@ public class TestReconstructStripedBlocks { try { fs.mkdirs(dirPath); fs.setErasureCodingPolicy(dirPath, - ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); + StripedFileTestUtil.getDefaultECPolicy().getName()); DFSTestUtil.createFile(fs, filePath, cellSize * dataBlocks * 2, (short) 1, 0L); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java index ae9793a6be9..f1610b1839e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.NameNodeProxies; +import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy; import org.apache.hadoop.hdfs.protocol.ClientProtocol; @@ -321,7 +322,7 @@ public class TestStripedINodeFile { // set erasure coding policy dfs.setErasureCodingPolicy(ecDir, - ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); + StripedFileTestUtil.getDefaultECPolicy().getName()); DFSTestUtil.createFile(dfs, ecFile, len, (short) 1, 0xFEED); DFSTestUtil.createFile(dfs, contiguousFile, len, (short) 1, 0xFEED); final FSDirectory fsd = fsn.getFSDirectory(); @@ -423,7 +424,7 @@ public class TestStripedINodeFile { client.setStoragePolicy(fooDir, HdfsConstants.ONESSD_STORAGE_POLICY_NAME); // set an EC policy on "/foo" directory client.setErasureCodingPolicy(fooDir, - ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); + StripedFileTestUtil.getDefaultECPolicy().getName()); // write file to fooDir final String barFile = "/foo/bar"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java index e7794d6c9ec..0bfa0543026 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java @@ -31,12 +31,12 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped; -import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.hdfs.server.namenode.FSDirectory; import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; import org.apache.hadoop.hdfs.server.namenode.INodeFile; @@ -46,7 +46,7 @@ import org.junit.Test; public class TestOfflineImageViewerWithStripedBlocks { private final ErasureCodingPolicy ecPolicy = - ErasureCodingPolicyManager.getSystemDefaultPolicy(); + StripedFileTestUtil.getDefaultECPolicy(); private int dataBlocks = ecPolicy.getNumDataUnits(); private int parityBlocks = ecPolicy.getNumParityUnits(); @@ -64,7 +64,7 @@ public class TestOfflineImageViewerWithStripedBlocks { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); cluster.waitActive(); cluster.getFileSystem().getClient().setErasureCodingPolicy("/", - ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); + StripedFileTestUtil.getDefaultECPolicy().getName()); fs = cluster.getFileSystem(); Path eczone = new Path("/eczone"); fs.mkdirs(eczone); @@ -144,7 +144,7 @@ public class TestOfflineImageViewerWithStripedBlocks { // Verify space consumed present in BlockInfoStriped FSDirectory fsdir = cluster.getNamesystem().getFSDirectory(); INodeFile fileNode = fsdir.getINode4Write(file.toString()).asFile(); - assertEquals(ErasureCodingPolicyManager.getSystemDefaultPolicy().getId(), + assertEquals(StripedFileTestUtil.getDefaultECPolicy().getId(), fileNode.getErasureCodingPolicyID()); assertTrue("Invalid block size", fileNode.getBlocks().length > 0); long actualFileSize = 0; From 846a0cd678fba743220f28cef844ac9011a3f934 Mon Sep 17 00:00:00 2001 From: Daniel Templeton Date: Thu, 9 Mar 2017 17:51:47 -0800 Subject: [PATCH 045/188] YARN-1047. Expose # of pre-emptions as a queue counter (Contributed by Karthik Kambatla via Daniel Templeton) --- .../resourcemanager/scheduler/QueueMetrics.java | 13 +++++++++++++ .../scheduler/fair/FSAppAttempt.java | 5 +++++ .../scheduler/fair/TestFairSchedulerPreemption.java | 12 +++++++++--- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java index 4e364f70f52..007d2b3b9fa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java @@ -71,6 +71,8 @@ public class QueueMetrics implements MetricsSource { @Metric("Aggregate # of allocated off-switch containers") MutableCounterLong aggregateOffSwitchContainersAllocated; @Metric("Aggregate # of released containers") MutableCounterLong aggregateContainersReleased; + @Metric("Aggregate # of preempted containers") MutableCounterLong + aggregateContainersPreempted; @Metric("Available memory in MB") MutableGaugeLong availableMB; @Metric("Available CPU in virtual cores") MutableGaugeInt availableVCores; @Metric("Pending memory allocation in MB") MutableGaugeLong pendingMB; @@ -476,6 +478,13 @@ public class QueueMetrics implements MetricsSource { } } + public void preemptContainer() { + aggregateContainersPreempted.incr(); + if (parent != null) { + parent.preemptContainer(); + } + } + public void reserveResource(String user, Resource res) { reservedContainers.incr(); reservedMB.incr(res.getMemorySize()); @@ -640,4 +649,8 @@ public class QueueMetrics implements MetricsSource { public long getAggegatedReleasedContainers() { return aggregateContainersReleased.value(); } + + public long getAggregatePreemptedContainers() { + return aggregateContainersPreempted.value(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java index 6c61b451d6d..3a9c94e6568 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java @@ -45,6 +45,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.PendingAsk; import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey; import org.apache.hadoop.yarn.server.utils.BuilderUtils; @@ -161,6 +162,10 @@ public class FSAppAttempt extends SchedulerApplicationAttempt } untrackContainerForPreemption(rmContainer); + if (containerStatus.getDiagnostics(). + equals(SchedulerUtils.PREEMPTED_CONTAINER)) { + queue.getMetrics().preemptContainer(); + } Resource containerResource = rmContainer.getContainer().getResource(); RMAuditLogger.logSuccess(getUser(), AuditConstants.RELEASE_CONTAINER, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java index 322ad5b3f58..3940a47aded 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java @@ -284,14 +284,20 @@ public class TestFairSchedulerPreemption extends FairSchedulerTestBase { Thread.sleep(10); } - // Verify the right amount of containers are preempted from greedyApp - assertEquals("Incorrect number of containers on the greedy app", + // Post preemption, verify the greedyApp has the correct # of containers. + assertEquals("Incorrect # of containers on the greedy app", 2 * numStarvedAppContainers, greedyApp.getLiveContainers().size()); + // Verify the queue metrics are set appropriately. The greedyApp started + // with 8 1GB, 1vcore containers. + assertEquals("Incorrect # of preempted containers in QueueMetrics", + 8 - 2 * numStarvedAppContainers, + greedyApp.getQueue().getMetrics().getAggregatePreemptedContainers()); + sendEnoughNodeUpdatesToAssignFully(); // Verify the preempted containers are assigned to starvingApp - assertEquals("Starved app is not assigned the right number of containers", + assertEquals("Starved app is not assigned the right # of containers", numStarvedAppContainers, starvingApp.getLiveContainers().size()); } From c5ee7fded46dcb1ac1ea4c1ada4949c50bc89afb Mon Sep 17 00:00:00 2001 From: John Zhuge Date: Sun, 5 Mar 2017 22:34:22 -0800 Subject: [PATCH 046/188] HADOOP-14123. Remove misplaced ADL service provider config file for FileSystem. Contributed by John Zhuge. Change-Id: Ic956e2eb8189625916442eaffdc69163d32f730e --- .../META-INF/org.apache.hadoop.fs.FileSystem | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 hadoop-tools/hadoop-azure-datalake/src/main/resources/META-INF/org.apache.hadoop.fs.FileSystem diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/resources/META-INF/org.apache.hadoop.fs.FileSystem b/hadoop-tools/hadoop-azure-datalake/src/main/resources/META-INF/org.apache.hadoop.fs.FileSystem deleted file mode 100644 index 7ec7812295d..00000000000 --- a/hadoop-tools/hadoop-azure-datalake/src/main/resources/META-INF/org.apache.hadoop.fs.FileSystem +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -org.apache.hadoop.fs.adl.AdlFileSystem \ No newline at end of file From a96afae125ba02fb4480542d3fb0891623ee4c37 Mon Sep 17 00:00:00 2001 From: Karthik Kambatla Date: Thu, 9 Mar 2017 23:11:54 -0800 Subject: [PATCH 047/188] YARN-6264. AM not launched when a single vcore is available on the cluster. (Yufei Gu via kasha) --- .../hadoop/yarn/util/resource/Resources.java | 7 +++++ .../yarn/util/resource/TestResources.java | 24 +++++++++++++++- .../scheduler/fair/FSLeafQueue.java | 3 +- .../scheduler/fair/TestFairScheduler.java | 28 +++++++++---------- 4 files changed, 46 insertions(+), 16 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index 57b3a463486..702030030ce 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -242,6 +242,13 @@ public class Resources { out.setVirtualCores((int)(lhs.getVirtualCores() * by)); return out; } + + public static Resource multiplyAndRoundUp(Resource lhs, double by) { + Resource out = clone(lhs); + out.setMemorySize((long)Math.ceil(lhs.getMemorySize() * by)); + out.setVirtualCores((int)Math.ceil(lhs.getVirtualCores() * by)); + return out; + } public static Resource normalize( ResourceCalculator calculator, Resource lhs, Resource min, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java index 057214badae..f8570a8a2f6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java @@ -20,6 +20,8 @@ package org.apache.hadoop.yarn.util.resource; import org.apache.hadoop.yarn.api.records.Resource; import org.junit.Test; + +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; public class TestResources { @@ -46,5 +48,25 @@ public class TestResources { assertTrue(Resources.none().compareTo( createResource(0, 1)) < 0); } - + + @Test + public void testMultipleRoundUp() { + final double by = 0.5; + final String memoryErrorMsg = "Invalid memory size."; + final String vcoreErrorMsg = "Invalid virtual core number."; + Resource resource = Resources.createResource(1, 1); + Resource result = Resources.multiplyAndRoundUp(resource, by); + assertEquals(memoryErrorMsg, result.getMemorySize(), 1); + assertEquals(vcoreErrorMsg, result.getVirtualCores(), 1); + + resource = Resources.createResource(2, 2); + result = Resources.multiplyAndRoundUp(resource, by); + assertEquals(memoryErrorMsg, result.getMemorySize(), 1); + assertEquals(vcoreErrorMsg, result.getVirtualCores(), 1); + + resource = Resources.createResource(0, 0); + result = Resources.multiplyAndRoundUp(resource, by); + assertEquals(memoryErrorMsg, result.getMemorySize(), 0); + assertEquals(vcoreErrorMsg, result.getVirtualCores(), 0); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java index aad291619cf..5be4b361afc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java @@ -515,7 +515,8 @@ public class FSLeafQueue extends FSQueue { getMaxShare().getVirtualCores())); } - return Resources.multiply(maxResource, maxAMShare); + // Round up to allow AM to run when there is only one vcore on the cluster + return Resources.multiplyAndRoundUp(maxResource, maxAMShare); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 028eea61829..cdf0d81fe2a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -659,15 +659,13 @@ public class TestFairScheduler extends FairSchedulerTestBase { // case, we use maxShare, since it is smaller than available resource. assertEquals("QueueFSZeroWithMax's fair share should be zero", 0, queueFSZeroWithMax.getFairShare().getMemorySize()); + Resource expectedAMResource = Resources.multiplyAndRoundUp( + queueFSZeroWithMax.getMaxShare(), queueFSZeroWithMax.getMaxAMShare()); assertEquals("QueueFSZeroWithMax's maximum AM resource should be " - + "maxShare * maxAMShare", - (long)(queueFSZeroWithMax.getMaxShare().getMemorySize() * - queueFSZeroWithMax.getMaxAMShare()), + + "maxShare * maxAMShare", expectedAMResource.getMemorySize(), queueFSZeroWithMax.getMetrics().getMaxAMShareMB()); assertEquals("QueueFSZeroWithMax's maximum AM resource should be " - + "maxShare * maxAMShare", - (long)(queueFSZeroWithMax.getMaxShare().getVirtualCores() * - queueFSZeroWithMax.getMaxAMShare()), + + "maxShare * maxAMShare", expectedAMResource.getVirtualCores(), queueFSZeroWithMax.getMetrics().getMaxAMShareVCores()); assertEquals("QueueFSZeroWithMax's AM resource usage should be the same to " + "AM resource request", @@ -689,17 +687,19 @@ public class TestFairScheduler extends FairSchedulerTestBase { // the min(maxShare, available resource) to compute maxAMShare, in this // case, we use available resource since it is smaller than the // default maxShare. + expectedAMResource = Resources.multiplyAndRoundUp( + Resources.createResource(memCapacity - amResource.getMemorySize(), + cpuCapacity - amResource.getVirtualCores()), + queueFSZeroWithAVL.getMaxAMShare()); assertEquals("QueueFSZeroWithAVL's fair share should be zero", 0, queueFSZeroWithAVL.getFairShare().getMemorySize()); assertEquals("QueueFSZeroWithAVL's maximum AM resource should be " + " available resource * maxAMShare", - (long) ((memCapacity - amResource.getMemorySize()) * - queueFSZeroWithAVL.getMaxAMShare()), + expectedAMResource.getMemorySize(), queueFSZeroWithAVL.getMetrics().getMaxAMShareMB()); assertEquals("QueueFSZeroWithAVL's maximum AM resource should be " + " available resource * maxAMShare", - (long) ((cpuCapacity - amResource.getVirtualCores()) * - queueFSZeroWithAVL.getMaxAMShare()), + expectedAMResource.getVirtualCores(), queueFSZeroWithAVL.getMetrics().getMaxAMShareVCores()); assertEquals("QueueFSZeroWithMax's AM resource usage should be the same to " + "AM resource request", @@ -721,13 +721,13 @@ public class TestFairScheduler extends FairSchedulerTestBase { // fair share to compute maxAMShare assertNotEquals("QueueFSNonZero's fair share shouldn't be zero", 0, queueFSNonZero.getFairShare().getMemorySize()); + expectedAMResource = Resources.multiplyAndRoundUp( + queueFSNonZero.getFairShare(), queueFSNonZero.getMaxAMShare()); assertEquals("QueueFSNonZero's maximum AM resource should be " - + " fair share * maxAMShare", - (long)(memCapacity * queueFSNonZero.getMaxAMShare()), + + " fair share * maxAMShare", expectedAMResource.getMemorySize(), queueFSNonZero.getMetrics().getMaxAMShareMB()); assertEquals("QueueFSNonZero's maximum AM resource should be " - + " fair share * maxAMShare", - (long)(cpuCapacity * queueFSNonZero.getMaxAMShare()), + + " fair share * maxAMShare", expectedAMResource.getVirtualCores(), queueFSNonZero.getMetrics().getMaxAMShareVCores()); assertEquals("QueueFSNonZero's AM resource usage should be the same to " + "AM resource request", From 881ec4d97bd1db4582027aec3a4204156a4eda17 Mon Sep 17 00:00:00 2001 From: Mingliang Liu Date: Tue, 7 Mar 2017 16:29:19 -0800 Subject: [PATCH 048/188] HADOOP-14153. ADL module has messed doc structure. Contributed by Mingliang Liu --- .../src/site/markdown/index.md | 53 +++++++------------ 1 file changed, 20 insertions(+), 33 deletions(-) diff --git a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md index 935524199b3..3a1625375be 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md +++ b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md @@ -14,28 +14,15 @@ # Hadoop Azure Data Lake Support -* [Introduction](#Introduction) -* [Features](#Features) -* [Limitations](#Limitations) -* [Usage](#Usage) - * [Concepts](#Concepts) - * [OAuth2 Support](#OAuth2_Support) - * [Configuring Credentials and FileSystem](#Configuring_Credentials) - * [Using Refresh Token](#Refresh_Token) - * [Using Client Keys](#Client_Credential_Token) - * [Protecting the Credentials with Credential Providers](#Credential_Provider) - * [Enabling ADL Filesystem](#Enabling_ADL) - * [Accessing `adl` URLs](#Accessing_adl_URLs) - * [User/Group Representation](#OIDtoUPNConfiguration) -* [Testing the `hadoop-azure` Module](#Testing_the_hadoop-azure_Module) + -## Introduction +## Introduction The `hadoop-azure-datalake` module provides support for integration with the [Azure Data Lake Store](https://azure.microsoft.com/en-in/documentation/services/data-lake-store/). This support comes via the JAR file `azure-datalake-store.jar`. -## Features +## Features * Read and write data stored in an Azure Data Lake Storage account. * Reference file system paths using URLs using the `adl` scheme for Secure Webhdfs i.e. SSL @@ -46,7 +33,7 @@ This support comes via the JAR file `azure-datalake-store.jar`. * API `setOwner()`, `setAcl`, `removeAclEntries()`, `modifyAclEntries()` accepts UPN or OID (Object ID) as user and group names. -## Limitations +## Limitations Partial or no support for the following operations : @@ -62,9 +49,9 @@ Partial or no support for the following operations : * User and group information returned as `listStatus()` and `getFileStatus()` is in the form of the GUID associated in Azure Active Directory. -## Usage +## Usage -### Concepts +### Concepts Azure Data Lake Storage access path syntax is: ``` @@ -74,7 +61,7 @@ adl://.azuredatalakestore.net/ For details on using the store, see [**Get started with Azure Data Lake Store using the Azure Portal**](https://azure.microsoft.com/en-in/documentation/articles/data-lake-store-get-started-portal/) -### OAuth2 Support +#### OAuth2 Support Usage of Azure Data Lake Storage requires an OAuth2 bearer token to be present as part of the HTTPS header as per the OAuth2 specification. @@ -86,11 +73,11 @@ and identity management service. See [*What is ActiveDirectory*](https://azure.m Following sections describes theOAuth2 configuration in `core-site.xml`. -#### Configuring Credentials & FileSystem +### Configuring Credentials and FileSystem Credentials can be configured using either a refresh token (associated with a user), or a client credential (analogous to a service principal). -#### Using Refresh Tokens +#### Using Refresh Tokens Add the following properties to the cluster's `core-site.xml` @@ -119,9 +106,9 @@ service associated with the client id. See [*Active Directory Library For Java*] ``` -### Using Client Keys +#### Using Client Keys -#### Generating the Service Principal +##### Generating the Service Principal 1. Go to [the portal](https://portal.azure.com) 2. Under "Browse", look for Active Directory and click on it. @@ -135,13 +122,13 @@ service associated with the client id. See [*Active Directory Library For Java*] - The token endpoint (select "View endpoints" at the bottom of the page and copy/paste the OAuth2 .0 Token Endpoint value) - Resource: Always https://management.core.windows.net/ , for all customers -#### Adding the service principal to your ADL Account +##### Adding the service principal to your ADL Account 1. Go to the portal again, and open your ADL account 2. Select Users under Settings 3. Add your user name you created in Step 6 above (note that it does not show up in the list, but will be found if you searched for the name) 4. Add "Owner" role -### Configure core-site.xml +##### Configure core-site.xml Add the following properties to your `core-site.xml` ```xml @@ -161,7 +148,7 @@ Add the following properties to your `core-site.xml` ``` -### Protecting the Credentials with Credential Providers +#### Protecting the Credentials with Credential Providers In many Hadoop clusters, the `core-site.xml` file is world-readable. To protect these credentials, it is recommended that you use the @@ -171,7 +158,7 @@ All ADLS credential properties can be protected by credential providers. For additional reading on the credential provider API, see [Credential Provider API](../hadoop-project-dist/hadoop-common/CredentialProviderAPI.html). -#### Provisioning +##### Provisioning ```bash hadoop credential create dfs.adls.oauth2.refresh.token -value 123 @@ -180,7 +167,7 @@ hadoop credential create dfs.adls.oauth2.credential -value 123 -provider localjceks://file/home/foo/adls.jceks ``` -#### Configuring core-site.xml or command line property +##### Configuring core-site.xml or command line property ```xml @@ -190,7 +177,7 @@ hadoop credential create dfs.adls.oauth2.credential -value 123 ``` -#### Running DistCp +##### Running DistCp ```bash hadoop distcp @@ -203,7 +190,7 @@ NOTE: You may optionally add the provider path property to the `distcp` command line instead of added job specific configuration to a generic `core-site.xml`. The square brackets above illustrate this capability.` -### Accessing adl URLs +### Accessing adl URLs After credentials are configured in `core-site.xml`, any Hadoop component may reference files in that Azure Data Lake Storage account by using URLs of the following @@ -230,7 +217,7 @@ hadoop fs -put testFile adl://yourcontainer.azuredatalakestore.net/testDir/testF hadoop fs -cat adl://yourcontainer.azuredatalakestore.net/testDir/testFile test file content ``` -### User/Group Representation +### User/Group Representation The `hadoop-azure-datalake` module provides support for configuring how User/Group information is represented during @@ -254,7 +241,7 @@ Add the following properties to `core-site.xml` ``` -## Testing the azure-datalake-store Module +## Testing the azure-datalake-store Module The `hadoop-azure` module includes a full suite of unit tests. Most of the tests will run without additional configuration by running mvn test. This includes tests against mocked storage, which is an in-memory emulation of Azure Data Lake Storage. From fd26783aaf3deea7a4e197439bd1075a6689681f Mon Sep 17 00:00:00 2001 From: Mingliang Liu Date: Fri, 10 Mar 2017 00:21:20 -0800 Subject: [PATCH 049/188] HADOOP-13946. Document how HDFS updates timestamps in the FS spec; compare with object stores. Contributed by Steve Loughran --- .../site/markdown/filesystem/introduction.md | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/introduction.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/introduction.md index f6db557c4a7..12a796717df 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/introduction.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/introduction.md @@ -392,3 +392,88 @@ Object stores with these characteristics, can not be used as a direct replacemen for HDFS. In terms of this specification, their implementations of the specified operations do not match those required. They are considered supported by the Hadoop development community, but not to the same extent as HDFS. + +#### Timestamps + + +`FileStatus` entries have a modification time and an access time. + +1. The exact behavior as to when these timestamps are set and whether or not they are valid +varies between filesystems, and potentially between individual installations of a filesystem. +1. The granularity of the timestamps is again, specific to both a filesystem +and potentially individual installations. + +The HDFS filesystem does not update the modification time while it is being written to. + +Specifically + +* `FileSystem.create()` creation: a zero-byte file is listed; the modification time is + set to the current time as seen on the NameNode. +* Writes to a file via the output stream returned in the `create()` call: the modification + time *does not change*. +* When `OutputStream.close()` is called, all remaining data is written, the file closed and + the NameNode updated with the final size of the file. The modification time is set to + the time the file was closed. +* Opening a file for appends via an `append()` operation does not change the modification + time of the file until the `close()` call is made on the output stream. +* `FileSystem.setTimes()` can be used to explicitly set the time on a file. +* When a file is renamed, its modification time is not changed, but the source + and destination directories have their modification times updated. +* The rarely used operations: `FileSystem.concat()`, `createSnapshot()`, + `createSymlink()` and `truncate()` all update the modification time. +* The access time granularity is set in milliseconds `dfs.namenode.access.time.precision`; + the default granularity is 1 hour. If the precision is set to zero, access times + are not recorded. +* If a modification or access time is not set, the value of that `FileStatus` +field is 0. + +Other filesystems may have different behaviors. In particular, + +* Access times may or may not be supported; even if the underlying FS may support access times, + the option it is often disabled for performance reasons. +* The granularity of the timestamps is an implementation-specific detail. + + +Object stores have an even vaguer view of time, which can be summarized as +"it varies". + + * The timestamp granularity is likely to be 1 second, that being the granularity + of timestamps returned in HTTP HEAD and GET requests. + * Access times are likely to be unset. That is, `FileStatus.getAccessTime() == 0`. + * The modification timestamp for a newly created file MAY be that of the + `create()` call, or the actual time which the PUT request was initiated. + This may be in the `FileSystem.create()` call, the final + `OutputStream.close()` operation, some period in between. + * The modification time may not be updated in the `close()` call. + * The timestamp is likely to be in UTC or the TZ of the object store. If the + client is in a different timezone, the timestamp of objects may be ahead or + behind that of the client. + * Object stores with cached metadata databases (for example: AWS S3 with + an in-memory or a DynamoDB metadata store) may have timestamps generated + from the local system clock, rather than that of the service. + This is an optimization to avoid round-trip calls to the object stores. + + A file's modification time is often the same as its creation time. + + The `FileSystem.setTimes()` operation to set file timestamps *may* be ignored. + * `FileSystem.chmod()` may update modification times (example: Azure `wasb://`). + * If `FileSystem.append()` is supported, the changes and modification time + are likely to only become visible after the output stream is closed. + * Out-of-band operations to data in object stores (that is: direct requests + to object stores which bypass the Hadoop FileSystem APIs), may result + in different timestamps being stored and/or returned. + * As the notion of a directory structure is often simulated, the timestamps + of directories *may* be artificially generated —perhaps using the current + system time. + * As `rename()` operations are often implemented as a COPY + DELETE, the + timestamps of renamed objects may become that of the time the rename of an + object was started, rather than the timestamp of the source object. + * The exact timestamp behavior may vary between different object store installations, + even with the same timestore client. + +Finally, note that the Apache Hadoop project cannot make any guarantees about +whether the timestamp behavior of a remote object store will remain consistent +over time: they are third-party services, usually accessed via third-party libraries. + +The best strategy here is "experiment with the exact endpoint you intend to work with". +Furthermore, if you intend to use any caching/consistency layer, test with that +feature enabled. Retest after updates to Hadoop releases, and endpoint object +store updates. From e06ff18ab68d23a0f236df8a0603a42367927f3c Mon Sep 17 00:00:00 2001 From: Sunil G Date: Fri, 10 Mar 2017 16:17:48 +0530 Subject: [PATCH 050/188] YARN-6196. Improve Resource Donut chart with better label in Node page of new YARN UI. Contributed by Akhil PB. --- .../webapp/app/helpers/log-files-comma.js | 10 +++++++- .../app/serializers/yarn-node-container.js | 2 +- .../main/webapp/app/serializers/yarn-node.js | 4 ++-- .../webapp/app/serializers/yarn-rm-node.js | 4 ++-- .../main/webapp/app/templates/yarn-node.hbs | 23 ++++++++++++------- .../main/webapp/app/templates/yarn-nodes.hbs | 2 +- .../webapp/app/templates/yarn-nodes/table.hbs | 13 ++++++----- 7 files changed, 37 insertions(+), 21 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/helpers/log-files-comma.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/helpers/log-files-comma.js index 78dcf257bf5..026cd7f9916 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/helpers/log-files-comma.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/helpers/log-files-comma.js @@ -35,8 +35,16 @@ export default Ember.Helper.helper(function(params,hash) { var containerId = hash.containerId; var html = ''; for (var i = 0; i < logFilesLen; i++) { + var logFileName = ""; + if (logFiles[i]) { + if (typeof logFiles[i] === "object" && logFiles[i].containerLogFiles) { + logFileName = logFiles[i].containerLogFiles; + } else if (typeof logFiles[i] === "string") { + logFileName = logFiles[i]; + } + } html = html + '' + logFiles[i] + + nodeAddr + '/' + containerId + '/' + logFileName + '">' + logFileName + ''; if (i !== logFilesLen - 1) { html = html + ","; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-node-container.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-node-container.js index 7e789879405..7bcb6553c0b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-node-container.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-node-container.js @@ -30,7 +30,7 @@ export default DS.JSONAPISerializer.extend({ containerId: payload.id, state: payload.state, user: payload.user, - diagnostics: payload.diagnostics, + diagnostics: payload.diagnostics || 'N/A', exitCode: payload.exitCode, totalMemoryNeeded: payload.totalMemoryNeededMB, totalVCoresNeeded: payload.totalVCoresNeeded, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-node.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-node.js index 0d9faec7231..10521e62a83 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-node.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-node.js @@ -36,8 +36,8 @@ export default DS.JSONAPISerializer.extend({ pmemCheckEnabled: payload.pmemCheckEnabled, nodeHealthy: payload.nodeHealthy, lastNodeUpdateTime: Converter.timeStampToDate(payload.lastNodeUpdateTime), - healthReport: payload.healthReport, - nmStartupTime: Converter.timeStampToDate(payload.nmStartupTime), + healthReport: payload.healthReport || 'N/A', + nmStartupTime: payload.nmStartupTime? Converter.timeStampToDate(payload.nmStartupTime) : '', nodeManagerBuildVersion: payload.nodeManagerBuildVersion, hadoopBuildVersion: payload.hadoopBuildVersion } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js index ad5062170b5..1c6d1be859a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js @@ -34,7 +34,7 @@ export default DS.JSONAPISerializer.extend({ nodeHostName: payload.nodeHostName, nodeHTTPAddress: payload.nodeHTTPAddress, lastHealthUpdate: Converter.timeStampToDate(payload.lastHealthUpdate), - healthReport: payload.healthReport, + healthReport: payload.healthReport || 'N/A', numContainers: payload.numContainers, usedMemoryMB: payload.usedMemoryMB, availMemoryMB: payload.availMemoryMB, @@ -57,7 +57,7 @@ export default DS.JSONAPISerializer.extend({ normalizeArrayResponse(store, primaryModelClass, payload/*, id, requestType*/) { // expected response is of the form { data: [ {}, {} ] } var normalizedArrayResponse = {}; - if (payload.nodes) { + if (payload.nodes && payload.nodes.node) { // payload is of the form { "nodes": { "node": [ {},{},{} ] } } normalizedArrayResponse.data = payload.nodes.node.map(singleNode => { return this.internalNormalizeSingleResponse(store, primaryModelClass, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node.hbs index b7f727af726..1e8549bd87f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node.hbs +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node.hbs @@ -28,7 +28,7 @@
-
Node Information
+
Node Information: {{model.rmNode.id}}
@@ -63,10 +63,12 @@ - - - - + {{#if model.node.nmStartupTime}} + + + + + {{/if}} @@ -82,22 +84,25 @@
-
+
- Resource - Memory (in MB) + Resource - Memory
{{donut-chart data=model.rmNode.getMemoryDataForDonutChart showLabels=true parentId="mem-donut-chart" ratio=0.6 + type="memory" + colorTargets="good" + colorTargetReverse=true maxHeight=350}}
-
+
Resource - VCores @@ -107,6 +112,8 @@ showLabels=true parentId="vcore-donut-chart" ratio=0.6 + colorTargets="good" + colorTargetReverse=true maxHeight=350}}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-nodes.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-nodes.hbs index 874b3c46f82..795d00efb11 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-nodes.hbs +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-nodes.hbs @@ -29,7 +29,7 @@
    {{#link-to "yarn-nodes.table" tagName="li"}} - {{#link-to 'yarn-nodes.table'}}Nodes Table + {{#link-to 'yarn-nodes.table'}}Information {{/link-to}} {{/link-to}} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-nodes/table.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-nodes/table.hbs index 1390906f84c..79d721bfc4e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-nodes/table.hbs +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-nodes/table.hbs @@ -16,12 +16,13 @@ limitations under the License. --}}
    - - {{#if model.nodes}} - {{em-table columns=columns rows=model.nodes}} - {{else}} -

    No nodes found on this cluster

    - {{/if}} +
    + {{#if model.nodes}} + {{em-table columns=columns rows=model.nodes}} + {{else}} +

    No nodes found on this cluster

    + {{/if}} +
    {{outlet}} From deb9f569465bb760e661e60a313dad1605635236 Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Fri, 10 Mar 2017 11:07:19 -0600 Subject: [PATCH 051/188] YARN-6310. OutputStreams in AggregatedLogFormat.LogWriter can be left open upon exceptions. Contributed by Haibo Chen --- .../logaggregation/AggregatedLogFormat.java | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java index 02f7782ebfd..1b46007ebb9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java @@ -486,34 +486,34 @@ public class AggregatedLogFormat { } private void writeVersion() throws IOException { - DataOutputStream out = this.writer.prepareAppendKey(-1); - VERSION_KEY.write(out); - out.close(); - out = this.writer.prepareAppendValue(-1); - out.writeInt(VERSION); - out.close(); + try (DataOutputStream out = this.writer.prepareAppendKey(-1)) { + VERSION_KEY.write(out); + } + try (DataOutputStream out = this.writer.prepareAppendValue(-1)) { + out.writeInt(VERSION); + } } public void writeApplicationOwner(String user) throws IOException { - DataOutputStream out = this.writer.prepareAppendKey(-1); - APPLICATION_OWNER_KEY.write(out); - out.close(); - out = this.writer.prepareAppendValue(-1); - out.writeUTF(user); - out.close(); + try (DataOutputStream out = this.writer.prepareAppendKey(-1)) { + APPLICATION_OWNER_KEY.write(out); + } + try (DataOutputStream out = this.writer.prepareAppendValue(-1)) { + out.writeUTF(user); + } } public void writeApplicationACLs(Map appAcls) throws IOException { - DataOutputStream out = this.writer.prepareAppendKey(-1); - APPLICATION_ACL_KEY.write(out); - out.close(); - out = this.writer.prepareAppendValue(-1); - for (Entry entry : appAcls.entrySet()) { - out.writeUTF(entry.getKey().toString()); - out.writeUTF(entry.getValue()); + try (DataOutputStream out = this.writer.prepareAppendKey(-1)) { + APPLICATION_ACL_KEY.write(out); + } + try (DataOutputStream out = this.writer.prepareAppendValue(-1)) { + for (Entry entry : appAcls.entrySet()) { + out.writeUTF(entry.getKey().toString()); + out.writeUTF(entry.getValue()); + } } - out.close(); } public void append(LogKey logKey, LogValue logValue) throws IOException { @@ -522,12 +522,12 @@ public class AggregatedLogFormat { if (pendingUploadFiles.size() == 0) { return; } - DataOutputStream out = this.writer.prepareAppendKey(-1); - logKey.write(out); - out.close(); - out = this.writer.prepareAppendValue(-1); - logValue.write(out, pendingUploadFiles); - out.close(); + try (DataOutputStream out = this.writer.prepareAppendKey(-1)) { + logKey.write(out); + } + try (DataOutputStream out = this.writer.prepareAppendValue(-1)) { + logValue.write(out, pendingUploadFiles); + } } public void close() { From 4478273e5fb731de93ff12e249a3137c38fcf46f Mon Sep 17 00:00:00 2001 From: Jian He Date: Thu, 9 Mar 2017 19:28:09 -0800 Subject: [PATCH 052/188] HADOOP-14062. ApplicationMasterProtocolPBClientImpl.allocate fails with EOFException when RPC privacy is enabled. Contributed by Steven Rand --- .../java/org/apache/hadoop/ipc/Client.java | 4 +++- .../yarn/client/api/impl/TestAMRMClient.java | 18 +++++++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java index 70b902c7bb9..c0a5be95f9d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java @@ -1768,7 +1768,9 @@ public class Client implements AutoCloseable { } void setSaslClient(SaslRpcClient client) throws IOException { - setInputStream(client.getInputStream(in)); + // Wrap the input stream in a BufferedInputStream to fill the buffer + // before reading its length (HADOOP-14062). + setInputStream(new BufferedInputStream(client.getInputStream(in))); setOutputStream(client.getOutputStream(out)); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java index 43c02710f8c..06ba137d0c2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java @@ -40,6 +40,7 @@ import java.util.Set; import java.util.TreeSet; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.Credentials; @@ -126,8 +127,12 @@ public class TestAMRMClient { @Before public void setup() throws Exception { - // start minicluster conf = new YarnConfiguration(); + createClusterAndStartApplication(); + } + + private void createClusterAndStartApplication() throws Exception { + // start minicluster conf.set(YarnConfiguration.RM_SCHEDULER, schedulerName); conf.setLong( YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, @@ -866,6 +871,17 @@ public class TestAMRMClient { initAMRMClientAndTest(true); } + @Test (timeout=60000) + public void testAMRMClientWithSaslEncryption() throws Exception { + // we have to create a new instance of MiniYARNCluster to avoid SASL qop + // mismatches between client and server + teardown(); + conf = new YarnConfiguration(); + conf.set(CommonConfigurationKeysPublic.HADOOP_RPC_PROTECTION, "privacy"); + createClusterAndStartApplication(); + initAMRMClientAndTest(false); + } + private void initAMRMClientAndTest(boolean useAllocReqId) throws YarnException, IOException { AMRMClient amClient = null; From 092ec39fb9d9930d234ed1f0ec507b2f8c6ff4bc Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Fri, 10 Mar 2017 17:43:22 +0000 Subject: [PATCH 053/188] HADOOP-14111 cut some obsolete, ignored s3 tests in TestS3Credentials. Contributed by Yuanbo Liu --- .../hadoop/fs/s3native/TestS3Credentials.java | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestS3Credentials.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestS3Credentials.java index 33d032086c6..17b78c7a2b7 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestS3Credentials.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3native/TestS3Credentials.java @@ -28,7 +28,6 @@ import java.io.File; import java.net.URI; import org.junit.Before; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -127,21 +126,4 @@ public class TestS3Credentials { s3Credentials.getSecretAccessKey()); } - @Test(expected=IllegalArgumentException.class) - @Ignore - public void noSecretShouldThrow() throws Exception { - S3Credentials s3Credentials = new S3Credentials(); - Configuration conf = new Configuration(); - conf.set(S3_NATIVE_AWS_ACCESS_KEY_ID, EXAMPLE_ID); - s3Credentials.initialize(new URI("s3n://foobar"), conf); - } - - @Test(expected=IllegalArgumentException.class) - @Ignore - public void noAccessIdShouldThrow() throws Exception { - S3Credentials s3Credentials = new S3Credentials(); - Configuration conf = new Configuration(); - conf.set(S3_NATIVE_AWS_SECRET_ACCESS_KEY, EXAMPLE_KEY); - s3Credentials.initialize(new URI("s3n://foobar"), conf); - } } From 9649c27864a23ea156bae904368c1d3cf94c6e9d Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Fri, 10 Mar 2017 13:05:55 -0600 Subject: [PATCH 054/188] YARN-6321. TestResources test timeouts are too aggressive. Contributed by Eric Badger --- .../org/apache/hadoop/yarn/util/resource/TestResources.java | 6 +++--- .../yarn/server/resourcemanager/resource/TestResources.java | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java index f8570a8a2f6..d79179ac0d9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java @@ -30,7 +30,7 @@ public class TestResources { return Resource.newInstance(memory, vCores); } - @Test(timeout=1000) + @Test(timeout=10000) public void testCompareToWithUnboundedResource() { assertTrue(Resources.unbounded().compareTo( createResource(Long.MAX_VALUE, Integer.MAX_VALUE)) == 0); @@ -40,7 +40,7 @@ public class TestResources { createResource(0, Integer.MAX_VALUE)) > 0); } - @Test(timeout=1000) + @Test(timeout=10000) public void testCompareToWithNoneResource() { assertTrue(Resources.none().compareTo(createResource(0, 0)) == 0); assertTrue(Resources.none().compareTo( @@ -49,7 +49,7 @@ public class TestResources { createResource(0, 1)) < 0); } - @Test + @Test(timeout=10000) public void testMultipleRoundUp() { final double by = 0.5; final String memoryErrorMsg = "Invalid memory size."; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResources.java index ae98660688a..2a10747ac9d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResources.java @@ -22,7 +22,7 @@ import static org.junit.Assert.*; import org.junit.Test; public class TestResources { - @Test(timeout=1000) + @Test(timeout=10000) public void testFitsIn() { assertTrue(fitsIn(createResource(1, 1), createResource(2, 2))); assertTrue(fitsIn(createResource(2, 2), createResource(2, 2))); @@ -31,7 +31,7 @@ public class TestResources { assertFalse(fitsIn(createResource(2, 1), createResource(1, 2))); } - @Test(timeout=1000) + @Test(timeout=10000) public void testComponentwiseMin() { assertEquals(createResource(1, 1), componentwiseMin(createResource(1, 1), createResource(2, 2))); From 6d356b6b4d8ccb32397cacfb5d0357b21f6035fc Mon Sep 17 00:00:00 2001 From: Lei Xu Date: Fri, 10 Mar 2017 14:36:51 -0800 Subject: [PATCH 055/188] HDFS-11340. DataNode reconfigure for disks doesn't remove the failed volumes. (Manoj Govindassamy via lei) --- .../hadoop/hdfs/server/datanode/DataNode.java | 73 ++++++++--- .../fsdataset/impl/FsDatasetImpl.java | 14 ++- .../datanode/fsdataset/impl/FsVolumeList.java | 13 +- .../TestDataNodeVolumeFailureReporting.java | 116 ++++++++++++++++-- 4 files changed, 184 insertions(+), 32 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 6f24858a8d2..5a82850c3f1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -652,48 +652,84 @@ public class DataNode extends ReconfigurableBase ChangedVolumes parseChangedVolumes(String newVolumes) throws IOException { Configuration conf = new Configuration(); conf.set(DFS_DATANODE_DATA_DIR_KEY, newVolumes); - List locations = getStorageLocations(conf); + List newStorageLocations = getStorageLocations(conf); - if (locations.isEmpty()) { + if (newStorageLocations.isEmpty()) { throw new IOException("No directory is specified."); } - // Use the existing StorageLocation to detect storage type changes. - Map existingLocations = new HashMap<>(); + // Use the existing storage locations from the current conf + // to detect new storage additions or removals. + Map existingStorageLocations = new HashMap<>(); for (StorageLocation loc : getStorageLocations(getConf())) { - existingLocations.put(loc.getNormalizedUri().toString(), loc); + existingStorageLocations.put(loc.getNormalizedUri().toString(), loc); } ChangedVolumes results = new ChangedVolumes(); - results.newLocations.addAll(locations); + results.newLocations.addAll(newStorageLocations); for (Iterator it = storage.dirIterator(); it.hasNext(); ) { Storage.StorageDirectory dir = it.next(); boolean found = false; - for (Iterator sl = results.newLocations.iterator(); - sl.hasNext(); ) { - StorageLocation location = sl.next(); - if (location.matchesStorageDirectory(dir)) { - sl.remove(); - StorageLocation old = existingLocations.get( - location.getNormalizedUri().toString()); - if (old != null && - old.getStorageType() != location.getStorageType()) { + for (Iterator newLocationItr = + results.newLocations.iterator(); newLocationItr.hasNext();) { + StorageLocation newLocation = newLocationItr.next(); + if (newLocation.matchesStorageDirectory(dir)) { + StorageLocation oldLocation = existingStorageLocations.get( + newLocation.getNormalizedUri().toString()); + if (oldLocation != null && + oldLocation.getStorageType() != newLocation.getStorageType()) { throw new IOException("Changing storage type is not allowed."); } - results.unchangedLocations.add(location); + // Update the unchanged locations as this location + // from the new conf is really not a new one. + newLocationItr.remove(); + results.unchangedLocations.add(newLocation); found = true; break; } } + // New conf doesn't have the storage location which available in + // the current storage locations. Add to the deactivateLocations list. if (!found) { + LOG.info("Deactivation request received for active volume: " + + dir.getRoot().toString()); results.deactivateLocations.add( StorageLocation.parse(dir.getRoot().toString())); } } + // Use the failed storage locations from the current conf + // to detect removals in the new conf. + if (getFSDataset().getNumFailedVolumes() > 0) { + for (String failedStorageLocation : getFSDataset() + .getVolumeFailureSummary().getFailedStorageLocations()) { + boolean found = false; + for (Iterator newLocationItr = + results.newLocations.iterator(); newLocationItr.hasNext();) { + StorageLocation newLocation = newLocationItr.next(); + if (newLocation.getNormalizedUri().toString().equals( + failedStorageLocation)) { + // The failed storage is being re-added. DataNode#refreshVolumes() + // will take care of re-assessing it. + found = true; + break; + } + } + + // New conf doesn't have this failed storage location. + // Add to the deactivate locations list. + if (!found) { + LOG.info("Deactivation request received for failed volume: " + + failedStorageLocation); + results.deactivateLocations.add(StorageLocation.parse( + failedStorageLocation)); + } + } + } + return results; } @@ -716,8 +752,9 @@ public class DataNode extends ReconfigurableBase } try { - if (numOldDataDirs + changedVolumes.newLocations.size() - - changedVolumes.deactivateLocations.size() <= 0) { + if (numOldDataDirs + getFSDataset().getNumFailedVolumes() + + changedVolumes.newLocations.size() + - changedVolumes.deactivateLocations.size() <= 0) { throw new IOException("Attempt to remove all volumes."); } if (!changedVolumes.newLocations.isEmpty()) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index aff19ce97e0..169e0e6118f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -502,8 +502,10 @@ class FsDatasetImpl implements FsDatasetSpi { */ @Override public void removeVolumes( - Collection storageLocationsToRemove, + final Collection storageLocsToRemove, boolean clearFailure) { + Collection storageLocationsToRemove = + new ArrayList<>(storageLocsToRemove); Map> blkToInvalidate = new HashMap<>(); List storageToRemove = new ArrayList<>(); try (AutoCloseableLock lock = datasetLock.acquire()) { @@ -541,6 +543,16 @@ class FsDatasetImpl implements FsDatasetSpi { } storageToRemove.add(sd.getStorageUuid()); + storageLocationsToRemove.remove(sdLocation); + } + } + + // A reconfigure can remove the storage location which is already + // removed when the failure was detected by DataNode#checkDiskErrorAsync. + // Now, lets remove this from the failed volume list. + if (clearFailure) { + for (StorageLocation storageLocToRemove : storageLocationsToRemove) { + volumes.removeVolumeFailureInfo(storageLocToRemove); } } setupAsyncLazyPersistThreads(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java index 64921d7fcd6..e7f02287f34 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java @@ -371,8 +371,15 @@ class FsVolumeList { } void addVolumeFailureInfo(VolumeFailureInfo volumeFailureInfo) { - volumeFailureInfos.put(volumeFailureInfo.getFailedStorageLocation(), - volumeFailureInfo); + // There could be redundant requests for adding the same failed + // volume because of repeated DataNode reconfigure with same list + // of volumes. Ignoring update on failed volume so as to preserve + // old failed capacity details in the map. + if (!volumeFailureInfos.containsKey(volumeFailureInfo + .getFailedStorageLocation())) { + volumeFailureInfos.put(volumeFailureInfo.getFailedStorageLocation(), + volumeFailureInfo); + } } private void addVolumeFailureInfo(FsVolumeImpl vol) { @@ -382,7 +389,7 @@ class FsVolumeList { vol.getCapacity())); } - private void removeVolumeFailureInfo(StorageLocation location) { + void removeVolumeFailureInfo(StorageLocation location) { volumeFailureInfos.remove(location); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureReporting.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureReporting.java index fbbc7f98a3c..a3850efc981 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureReporting.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureReporting.java @@ -25,16 +25,22 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.io.File; +import java.lang.management.ManagementFactory; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; import java.util.concurrent.TimeUnit; +import javax.management.MBeanServer; +import javax.management.ObjectName; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.ReconfigurationException; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSConfigKeys; @@ -51,6 +57,7 @@ import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.test.GenericTestUtils; import org.apache.log4j.Level; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -406,8 +413,8 @@ public class TestDataNodeVolumeFailureReporting { DataNodeTestUtils.triggerHeartbeat(dns.get(0)); DataNodeTestUtils.triggerHeartbeat(dns.get(1)); - checkFailuresAtDataNode(dns.get(0), 1, false, dn1Vol1.getAbsolutePath()); - checkFailuresAtDataNode(dns.get(1), 1, false, dn2Vol1.getAbsolutePath()); + checkFailuresAtDataNode(dns.get(0), 1, true, dn1Vol1.getAbsolutePath()); + checkFailuresAtDataNode(dns.get(1), 1, true, dn2Vol1.getAbsolutePath()); // Ensure we wait a sufficient amount of time. assert (WAIT_FOR_HEARTBEATS * 10) > WAIT_FOR_DEATH; @@ -415,9 +422,9 @@ public class TestDataNodeVolumeFailureReporting { // The NN reports two volume failures again. DFSTestUtil.waitForDatanodeStatus(dm, 3, 0, 2, origCapacity - (1*dnCapacity), WAIT_FOR_HEARTBEATS); - checkAggregateFailuresAtNameNode(false, 2); - checkFailuresAtNameNode(dm, dns.get(0), false, dn1Vol1.getAbsolutePath()); - checkFailuresAtNameNode(dm, dns.get(1), false, dn2Vol1.getAbsolutePath()); + checkAggregateFailuresAtNameNode(true, 2); + checkFailuresAtNameNode(dm, dns.get(0), true, dn1Vol1.getAbsolutePath()); + checkFailuresAtNameNode(dm, dns.get(1), true, dn2Vol1.getAbsolutePath()); // Reconfigure a third time with the failed volumes. Afterwards, we expect // the same volume failures to be reported. (No double-counting.) @@ -427,8 +434,8 @@ public class TestDataNodeVolumeFailureReporting { DataNodeTestUtils.triggerHeartbeat(dns.get(0)); DataNodeTestUtils.triggerHeartbeat(dns.get(1)); - checkFailuresAtDataNode(dns.get(0), 1, false, dn1Vol1.getAbsolutePath()); - checkFailuresAtDataNode(dns.get(1), 1, false, dn2Vol1.getAbsolutePath()); + checkFailuresAtDataNode(dns.get(0), 1, true, dn1Vol1.getAbsolutePath()); + checkFailuresAtDataNode(dns.get(1), 1, true, dn2Vol1.getAbsolutePath()); // Ensure we wait a sufficient amount of time. assert (WAIT_FOR_HEARTBEATS * 10) > WAIT_FOR_DEATH; @@ -436,9 +443,9 @@ public class TestDataNodeVolumeFailureReporting { // The NN reports two volume failures again. DFSTestUtil.waitForDatanodeStatus(dm, 3, 0, 2, origCapacity - (1*dnCapacity), WAIT_FOR_HEARTBEATS); - checkAggregateFailuresAtNameNode(false, 2); - checkFailuresAtNameNode(dm, dns.get(0), false, dn1Vol1.getAbsolutePath()); - checkFailuresAtNameNode(dm, dns.get(1), false, dn2Vol1.getAbsolutePath()); + checkAggregateFailuresAtNameNode(true, 2); + checkFailuresAtNameNode(dm, dns.get(0), true, dn1Vol1.getAbsolutePath()); + checkFailuresAtNameNode(dm, dns.get(1), true, dn2Vol1.getAbsolutePath()); // Replace failed volume with healthy volume and run reconfigure DataNode. // The failed volume information should be cleared. @@ -514,6 +521,95 @@ public class TestDataNodeVolumeFailureReporting { currentVersion.exists()); } + /** + * Verify DataNode NumFailedVolumes and FailedStorageLocations + * after hot swap out of failed volume. + */ + @Test + public void testHotSwapOutFailedVolumeAndReporting() + throws Exception { + final File dn0Vol1 = new File(dataDir, "data" + (2 * 0 + 1)); + final File dn0Vol2 = new File(dataDir, "data" + (2 * 0 + 2)); + final DataNode dn0 = cluster.getDataNodes().get(0); + final String oldDataDirs = dn0.getConf().get( + DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY); + + MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); + ObjectName mxbeanName = new ObjectName( + "Hadoop:service=DataNode,name=FSDatasetState-" + dn0.getDatanodeUuid()); + int numFailedVolumes = (int) mbs.getAttribute(mxbeanName, + "NumFailedVolumes"); + Assert.assertEquals(dn0.getFSDataset().getNumFailedVolumes(), + numFailedVolumes); + checkFailuresAtDataNode(dn0, 0, false, new String[] {}); + + // Fail dn0Vol1 first. + // Verify NumFailedVolumes and FailedStorageLocations are empty. + DataNodeTestUtils.injectDataDirFailure(dn0Vol1); + DataNodeTestUtils.waitForDiskError(dn0, + DataNodeTestUtils.getVolume(dn0, dn0Vol1)); + numFailedVolumes = (int) mbs.getAttribute(mxbeanName, "NumFailedVolumes"); + Assert.assertEquals(1, numFailedVolumes); + Assert.assertEquals(dn0.getFSDataset().getNumFailedVolumes(), + numFailedVolumes); + checkFailuresAtDataNode(dn0, 1, true, + new String[] {dn0Vol1.getAbsolutePath()}); + + // Reconfigure disks without fixing the failed disk. + // Verify NumFailedVolumes and FailedStorageLocations haven't changed. + try { + dn0.reconfigurePropertyImpl(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, + oldDataDirs); + fail("Reconfigure with failed disk should throw exception."); + } catch (ReconfigurationException e) { + Assert.assertTrue("Reconfigure exception doesn't have expected path!", + e.getCause().getMessage().contains(dn0Vol1.getAbsolutePath())); + } + numFailedVolumes = (int) mbs.getAttribute(mxbeanName, "NumFailedVolumes"); + Assert.assertEquals(1, numFailedVolumes); + Assert.assertEquals(dn0.getFSDataset().getNumFailedVolumes(), + numFailedVolumes); + checkFailuresAtDataNode(dn0, 1, true, + new String[] {dn0Vol1.getAbsolutePath()}); + + // Hot swap out the failed volume. + // Verify NumFailedVolumes and FailedStorageLocations are reset. + String dataDirs = dn0Vol2.getPath(); + dn0.reconfigurePropertyImpl(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, + dataDirs); + numFailedVolumes = (int) mbs.getAttribute(mxbeanName, "NumFailedVolumes"); + Assert.assertEquals(0, numFailedVolumes); + Assert.assertEquals(dn0.getFSDataset().getNumFailedVolumes(), + numFailedVolumes); + checkFailuresAtDataNode(dn0, 0, true, new String[] {}); + + // Fix failure volume dn0Vol1 and remount it back. + // Verify NumFailedVolumes and FailedStorageLocations are empty. + DataNodeTestUtils.restoreDataDirFromFailure(dn0Vol1); + dn0.reconfigurePropertyImpl(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, + oldDataDirs); + numFailedVolumes = (int) mbs.getAttribute(mxbeanName, "NumFailedVolumes"); + Assert.assertEquals(0, numFailedVolumes); + Assert.assertEquals(dn0.getFSDataset().getNumFailedVolumes(), + numFailedVolumes); + checkFailuresAtDataNode(dn0, 0, true, new String[] {}); + + // Fail dn0Vol2. + // Verify NumFailedVolumes and FailedStorageLocations are updated. + DataNodeTestUtils.injectDataDirFailure(dn0Vol2); + DataNodeTestUtils.waitForDiskError(dn0, + DataNodeTestUtils.getVolume(dn0, dn0Vol2)); + numFailedVolumes = (int) mbs.getAttribute(mxbeanName, "NumFailedVolumes"); + Assert.assertEquals(1, numFailedVolumes); + Assert.assertEquals(dn0.getFSDataset().getNumFailedVolumes(), + numFailedVolumes); + checkFailuresAtDataNode(dn0, 1, true, + new String[] {dn0Vol2.getAbsolutePath()}); + + // Verify DataNode tolerating one disk failure. + assertTrue(dn0.shouldRun()); + } + /** * Checks the NameNode for correct values of aggregate counters tracking failed * volumes across all DataNodes. From 229c7c9f8922f2b1bfd04b09b834e00d359046ff Mon Sep 17 00:00:00 2001 From: Varun Saxena Date: Sat, 11 Mar 2017 02:11:49 +0530 Subject: [PATCH 056/188] YARN-6069. CORS support in timeline v2 (Rohith Sharma K S via Varun Saxena) --- .../src/main/resources/yarn-default.xml | 11 +++++++++++ .../timelineservice/reader/TimelineReaderServer.java | 9 +++++++++ .../src/site/markdown/TimelineServiceV2.md | 9 +++++++++ 3 files changed, 29 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 645a342da57..727e2c9fbdb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3068,4 +3068,15 @@ 64 + + + Flag to enable cross-origin (CORS) support for timeline service v1.x or + Timeline Reader in timeline service v2. For timeline service v2, also add + org.apache.hadoop.security.HttpCrossOriginFilterInitializer to the + configuration hadoop.http.filter.initializers in core-site.xml. + + yarn.timeline-service.http-cross-origin.enabled + false + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java index 2835c1b2959..2faf4b61eec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java @@ -32,6 +32,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.http.lib.StaticUserWebFilter; +import org.apache.hadoop.security.HttpCrossOriginFilterInitializer; import org.apache.hadoop.service.CompositeService; import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.util.ReflectionUtils; @@ -134,6 +135,14 @@ public class TimelineReaderServer extends CompositeService { YarnConfiguration.TIMELINE_SERVICE_BIND_HOST, WebAppUtils.getTimelineReaderWebAppURL(conf)); LOG.info("Instantiating TimelineReaderWebApp at " + bindAddress); + boolean enableCorsFilter = conf.getBoolean( + YarnConfiguration.TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED, + YarnConfiguration.TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED_DEFAULT); + // setup CORS + if (enableCorsFilter) { + conf.setBoolean(HttpCrossOriginFilterInitializer.PREFIX + + HttpCrossOriginFilterInitializer.ENABLED_SUFFIX, true); + } try { HttpServer2.Builder builder = new HttpServer2.Builder() .setName("timeline") diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md index dc1680362bf..bcbe0b7728e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md @@ -142,6 +142,15 @@ New configuration parameters that are introduced with v.2 are marked bold. | **`yarn.timeline-service.hbase.coprocessor.app-final-value-retention-milliseconds`** | The setting that controls how long the final value of a metric of a completed app is retained before merging into the flow sum. Defaults to `259200000` (3 days). This should be set in the HBase cluster. | | **`yarn.rm.system-metrics-publisher.emit-container-events`** | The setting that controls whether yarn container metrics is published to the timeline server or not by RM. This configuration setting is for ATS V2. Defaults to `false`. | +#### Enabling CORS support +To enable cross-origin support (CORS) for the Timeline Service v.2, please set the following configuration parameters: + +In yarn-site.xml, set yarn.timeline-service.http-cross-origin.enabled to true. + +In core-site.xml, add org.apache.hadoop.security.HttpCrossOriginFilterInitializer to hadoop.http.filter.initializers. + +For more configurations used for cross-origin support, refer to [HttpAuthentication](../../hadoop-project-dist/hadoop-common/HttpAuthentication.html#CORS). Please note that yarn.timeline-service.http-cross-origin.enabled, if set to true, overrides hadoop.http.cross-origin.enabled. + ### Enabling Timeline Service v.2 From 4db9cc70d0178703fb28f451eb84d97f2bf63af8 Mon Sep 17 00:00:00 2001 From: Ray Chiang Date: Fri, 10 Mar 2017 16:13:31 -0800 Subject: [PATCH 057/188] YARN-6042. Dump scheduler and queue state information into FairScheduler DEBUG log. (Yufei Gu via rchiang) --- .../src/main/conf/log4j.properties | 9 +++ .../scheduler/fair/FSAppAttempt.java | 49 +++++++------ .../scheduler/fair/FSLeafQueue.java | 21 ++++++ .../scheduler/fair/FSParentQueue.java | 21 ++++++ .../scheduler/fair/FSQueue.java | 41 ++++++++++- .../scheduler/fair/FairScheduler.java | 28 +++++--- .../scheduler/fair/TestFairScheduler.java | 71 +++++++++++++++++++ 7 files changed, 206 insertions(+), 34 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties index b8c84e7b302..60267635dfe 100644 --- a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties +++ b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties @@ -321,3 +321,12 @@ log4j.appender.EWMA=org.apache.hadoop.yarn.util.Log4jWarningErrorMetricsAppender log4j.appender.EWMA.cleanupInterval=${yarn.ewma.cleanupInterval} log4j.appender.EWMA.messageAgeLimitSeconds=${yarn.ewma.messageAgeLimitSeconds} log4j.appender.EWMA.maxUniqueMessages=${yarn.ewma.maxUniqueMessages} + +# Fair scheduler requests log on state dump +log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.statedump=DEBUG,FSLOGGER +log4j.appender.FSLOGGER=org.apache.log4j.RollingFileAppender +log4j.appender.FSLOGGER.File=${hadoop.log.dir}/fairscheduler-statedump.log +log4j.appender.FSLOGGER.layout=org.apache.log4j.PatternLayout +log4j.appender.FSLOGGER.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +log4j.appender.FSLOGGER.MaxFileSize=${hadoop.log.maxfilesize} +log4j.appender.FSLOGGER.MaxBackupIndex=${hadoop.log.maxbackupindex} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java index 3a9c94e6568..ccfcffb83d7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java @@ -835,25 +835,27 @@ public class FSAppAttempt extends SchedulerApplicationAttempt return capability; } + if (LOG.isDebugEnabled()) { + LOG.debug("Resource request: " + capability + " exceeds the available" + + " resources of the node."); + } + // The desired container won't fit here, so reserve if (isReservable(capability) && reserve(pendingAsk.getPerAllocationResource(), node, reservedContainer, type, schedulerKey)) { - if (isWaitingForAMContainer()) { - updateAMDiagnosticMsg(capability, - " exceed the available resources of the node and the request is" - + " reserved"); + updateAMDiagnosticMsg(capability, " exceeds the available resources of " + + "the node and the request is reserved)"); + if (LOG.isDebugEnabled()) { + LOG.debug(getName() + "'s resource request is reserved."); } return FairScheduler.CONTAINER_RESERVED; } else { - if (isWaitingForAMContainer()) { - updateAMDiagnosticMsg(capability, - " exceed the available resources of the node and the request cannot" - + " be reserved"); - } + updateAMDiagnosticMsg(capability, " exceeds the available resources of " + + "the node and the request cannot be reserved)"); if (LOG.isDebugEnabled()) { - LOG.debug("Couldn't creating reservation for " + - getName() + ",at priority " + schedulerKey.getPriority()); + LOG.debug("Couldn't create reservation for app: " + getName() + + ", at priority " + schedulerKey.getPriority()); } return Resources.none(); } @@ -1034,10 +1036,9 @@ public class FSAppAttempt extends SchedulerApplicationAttempt ret = false; } else if (!getQueue().fitsInMaxShare(resource)) { // The requested container must fit in queue maximum share - if (isWaitingForAMContainer()) { - updateAMDiagnosticMsg(resource, - " exceeds current queue or its parents maximum resource allowed)."); - } + updateAMDiagnosticMsg(resource, + " exceeds current queue or its parents maximum resource allowed)."); + ret = false; } @@ -1309,15 +1310,13 @@ public class FSAppAttempt extends SchedulerApplicationAttempt @Override public Resource assignContainer(FSSchedulerNode node) { if (isOverAMShareLimit()) { - if (isWaitingForAMContainer()) { - PendingAsk amAsk = appSchedulingInfo.getNextPendingAsk(); - updateAMDiagnosticMsg(amAsk.getPerAllocationResource(), - " exceeds maximum AM resource allowed)."); - } - + PendingAsk amAsk = appSchedulingInfo.getNextPendingAsk(); + updateAMDiagnosticMsg(amAsk.getPerAllocationResource(), + " exceeds maximum AM resource allowed)."); if (LOG.isDebugEnabled()) { - LOG.debug("Skipping allocation because maxAMShare limit would " + - "be exceeded"); + LOG.debug("AM resource request: " + amAsk.getPerAllocationResource() + + " exceeds maximum AM resource allowed, " + + getQueue().dumpState()); } return Resources.none(); } @@ -1331,6 +1330,10 @@ public class FSAppAttempt extends SchedulerApplicationAttempt * @param reason the reason why AM doesn't get the resource */ private void updateAMDiagnosticMsg(Resource resource, String reason) { + if (!isWaitingForAMContainer()) { + return; + } + StringBuilder diagnosticMessageBldr = new StringBuilder(); diagnosticMessageBldr.append(" (Resource request: "); diagnosticMessageBldr.append(resource); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java index 5be4b361afc..10f1e287f41 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java @@ -617,4 +617,25 @@ public class FSLeafQueue extends FSQueue { boolean isStarved() { return isStarvedForMinShare() || isStarvedForFairShare(); } + + @Override + protected void dumpStateInternal(StringBuilder sb) { + sb.append("{Name: " + getName() + + ", Weight: " + weights + + ", Policy: " + policy.getName() + + ", FairShare: " + getFairShare() + + ", SteadyFairShare: " + getSteadyFairShare() + + ", MaxShare: " + maxShare + + ", MinShare: " + minShare + + ", ResourceUsage: " + getResourceUsage() + + ", Demand: " + getDemand() + + ", Runnable: " + getNumRunnableApps() + + ", NumPendingApps: " + getNumPendingApps() + + ", NonRunnable: " + getNumNonRunnableApps() + + ", MaxAMShare: " + maxAMShare + + ", MaxAMResource: " + computeMaxAMResource() + + ", AMResourceUsage: " + getAmResourceUsage() + + ", LastTimeAtMinShare: " + lastTimeAtMinShare + + "}"); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java index 1c8e9ced590..b062c586d01 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java @@ -292,4 +292,25 @@ public class FSParentQueue extends FSQueue { // TODO Auto-generated method stub } + + @Override + protected void dumpStateInternal(StringBuilder sb) { + sb.append("{Name: " + getName() + + ", Weight: " + weights + + ", Policy: " + policy.getName() + + ", FairShare: " + getFairShare() + + ", SteadyFairShare: " + getSteadyFairShare() + + ", MaxShare: " + maxShare + + ", MinShare: " + minShare + + ", ResourceUsage: " + getResourceUsage() + + ", Demand: " + getDemand() + + ", MaxAMShare: " + maxAMShare + + ", Runnable: " + getNumRunnableApps() + + "}"); + + for(FSQueue child : getChildQueues()) { + sb.append(", "); + child.dumpStateInternal(sb); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java index b5592c55bc2..acf4d5c203b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java @@ -393,11 +393,22 @@ public abstract class FSQueue implements Queue, Schedulable { * @return true if check passes (can assign) or false otherwise */ boolean assignContainerPreCheck(FSSchedulerNode node) { - if (!Resources.fitsIn(getResourceUsage(), maxShare) - || node.getReservedContainer() != null) { + if (node.getReservedContainer() != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Assigning container failed on node '" + node.getNodeName() + + " because it has reserved containers."); + } return false; + } else if (!Resources.fitsIn(getResourceUsage(), maxShare)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Assigning container failed on node '" + node.getNodeName() + + " because queue resource usage is larger than MaxShare: " + + dumpState()); + } + return false; + } else { + return true; } - return true; } /** @@ -453,6 +464,11 @@ public abstract class FSQueue implements Queue, Schedulable { Resources.add(getResourceUsage(), additionalResource); if (!Resources.fitsIn(usagePlusAddition, getMaxShare())) { + if (LOG.isDebugEnabled()) { + LOG.debug("Resource usage plus resource request: " + usagePlusAddition + + " exceeds maximum resource allowed:" + getMaxShare() + + " in queue " + getName()); + } return false; } @@ -491,4 +507,23 @@ public abstract class FSQueue implements Queue, Schedulable { setPolicy(queuePolicy); return true; } + + /** + * Recursively dump states of all queues. + * + * @return a string which holds all queue states + */ + public String dumpState() { + StringBuilder sb = new StringBuilder(); + dumpStateInternal(sb); + return sb.toString(); + } + + + /** + * Recursively dump states of all queues. + * + * @param sb the {code StringBuilder} which holds queue states + */ + protected abstract void dumpStateInternal(StringBuilder sb); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 4f3e4f9140e..f3fde76f06a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -139,7 +139,9 @@ public class FairScheduler extends private boolean usePortForNodeName; private static final Log LOG = LogFactory.getLog(FairScheduler.class); - + private static final Log STATE_DUMP_LOG = + LogFactory.getLog(FairScheduler.class.getName() + ".statedump"); + private static final ResourceCalculator RESOURCE_CALCULATOR = new DefaultResourceCalculator(); private static final ResourceCalculator DOMINANT_RESOURCE_CALCULATOR = @@ -151,7 +153,7 @@ public class FairScheduler extends // How often fair shares are re-calculated (ms) protected long updateInterval; - private final int UPDATE_DEBUG_FREQUENCY = 5; + private final int UPDATE_DEBUG_FREQUENCY = 25; private int updatesToSkipForDebug = UPDATE_DEBUG_FREQUENCY; @VisibleForTesting @@ -344,6 +346,21 @@ public class FairScheduler extends } } + /** + * Dump scheduler state including states of all queues. + */ + private void dumpSchedulerState() { + FSQueue rootQueue = queueMgr.getRootQueue(); + Resource clusterResource = getClusterResource(); + LOG.debug("FairScheduler state: Cluster Capacity: " + clusterResource + + " Allocations: " + rootMetrics.getAllocatedResources() + + " Availability: " + Resource.newInstance( + rootMetrics.getAvailableMB(), rootMetrics.getAvailableVirtualCores()) + + " Demand: " + rootQueue.getDemand()); + + STATE_DUMP_LOG.debug(rootQueue.dumpState()); + } + /** * Recompute the internal variables used by the scheduler - per-job weights, * fair shares, deficits, minimum slot allocations, and amount of used and @@ -368,12 +385,7 @@ public class FairScheduler extends if (LOG.isDebugEnabled()) { if (--updatesToSkipForDebug < 0) { updatesToSkipForDebug = UPDATE_DEBUG_FREQUENCY; - LOG.debug("Cluster Capacity: " + clusterResource + - " Allocations: " + rootMetrics.getAllocatedResources() + - " Availability: " + Resource.newInstance( - rootMetrics.getAvailableMB(), - rootMetrics.getAvailableVirtualCores()) + - " Demand: " + rootQueue.getDemand()); + dumpSchedulerState(); } } } finally { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index cdf0d81fe2a..537d3d0d392 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -5146,4 +5146,75 @@ public class TestFairScheduler extends FairSchedulerTestBase { Resources.equals(aQueue.getDemand(), maxResource) && Resources.equals(bQueue.getDemand(), maxResource)); } + + @Test + public void testDumpState() throws IOException { + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(" "); + out.println(" 1"); + out.println(" "); + out.println(""); + out.println(""); + out.close(); + + ControlledClock clock = new ControlledClock(); + scheduler.setClock(clock); + + scheduler.init(conf); + scheduler.start(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + FSLeafQueue child1 = + scheduler.getQueueManager().getLeafQueue("parent.child1", false); + Resource resource = Resource.newInstance(4 * GB, 4); + child1.setMaxShare(resource); + FSAppAttempt app = mock(FSAppAttempt.class); + Mockito.when(app.getDemand()).thenReturn(resource); + Mockito.when(app.getResourceUsage()).thenReturn(resource); + child1.addAppSchedulable(app); + child1.updateDemand(); + + String childQueueString = "{Name: root.parent.child1," + + " Weight: ," + + " Policy: fair," + + " FairShare: ," + + " SteadyFairShare: ," + + " MaxShare: ," + + " MinShare: ," + + " ResourceUsage: ," + + " Demand: ," + + " Runnable: 1," + + " NumPendingApps: 0," + + " NonRunnable: 0," + + " MaxAMShare: 0.5," + + " MaxAMResource: ," + + " AMResourceUsage: ," + + " LastTimeAtMinShare: " + clock.getTime() + + "}"; + + assertTrue(child1.dumpState().equals(childQueueString)); + FSParentQueue parent = + scheduler.getQueueManager().getParentQueue("parent", false); + parent.setMaxShare(resource); + + String parentQueueString = "{Name: root.parent," + + " Weight: ," + + " Policy: fair," + + " FairShare: ," + + " SteadyFairShare: ," + + " MaxShare: ," + + " MinShare: ," + + " ResourceUsage: ," + + " Demand: ," + + " MaxAMShare: 0.5," + + " Runnable: 0}"; + + assertTrue(parent.dumpState().equals( + parentQueueString + ", " + childQueueString)); + } } From 04a5f5a6dc88769cca8b1a15057a0756712b5013 Mon Sep 17 00:00:00 2001 From: Andrey Dyatlov Date: Mon, 6 Feb 2017 19:05:58 +0100 Subject: [PATCH 058/188] HADOOP-14156. Fix grammar error in ConfTest.java. This closes #187 Signed-off-by: Akira Ajisaka --- .../src/main/java/org/apache/hadoop/util/ConfTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ConfTest.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ConfTest.java index 3f37f5a7471..1915e791d21 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ConfTest.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ConfTest.java @@ -269,7 +269,7 @@ public final class ConfTest { } else { String confDirName = System.getenv(HADOOP_CONF_DIR); if (confDirName == null) { - terminate(1, HADOOP_CONF_DIR + " does not defined"); + terminate(1, HADOOP_CONF_DIR + " is not defined"); } File confDir = new File(confDirName); if (!confDir.isDirectory()) { From 79924266f8f68e5e7c873e6b12e3b3acfcd708da Mon Sep 17 00:00:00 2001 From: Yiqun Lin Date: Mon, 13 Mar 2017 18:22:30 +0800 Subject: [PATCH 059/188] HDFS-11512. Increase timeout on TestShortCircuitLocalRead#testSkipWithVerifyChecksum. Contributed by Eric Badger. --- .../hadoop/hdfs/shortcircuit/TestShortCircuitLocalRead.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitLocalRead.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitLocalRead.java index 55e97958791..f2ee48c6f0a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitLocalRead.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitLocalRead.java @@ -388,7 +388,7 @@ public class TestShortCircuitLocalRead { } } - @Test(timeout=10000) + @Test(timeout=60000) public void testSkipWithVerifyChecksum() throws IOException { int size = blockSize; Configuration conf = new Configuration(); From 5a40bafdaeec693e613aa02e79dbaaccfdab6f60 Mon Sep 17 00:00:00 2001 From: John Zhuge Date: Fri, 10 Mar 2017 17:42:30 -0800 Subject: [PATCH 060/188] HADOOP-14173. Remove unused AdlConfKeys#ADL_EVENTS_TRACKING_SOURCE. Contributed by John Zhuge. Change-Id: I5dc6f885816b8834f718874542dfa373458b0333 --- .../src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java index 7d31103d976..8fc8e00b84c 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java +++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java @@ -66,7 +66,6 @@ public final class AdlConfKeys { static final String ADL_HADOOP_CLIENT_NAME = "hadoop-azure-datalake-"; static final String ADL_HADOOP_CLIENT_VERSION = "2.0.0-SNAPSHOT"; - static final String ADL_EVENTS_TRACKING_SOURCE = "adl.events.tracking.source"; static final String ADL_EVENTS_TRACKING_CLUSTERNAME = "adl.events.tracking.clustername"; From b8c69557b7a23ff9c4c0b2c9d595338a08b873f1 Mon Sep 17 00:00:00 2001 From: Mingliang Liu Date: Fri, 10 Mar 2017 18:44:27 -0800 Subject: [PATCH 061/188] HADOOP-14170. FileSystemContractBaseTest is not cleaning up test directory clearly. Contributed by Mingliang Liu --- .../hadoop/fs/FileSystemContractBaseTest.java | 247 +++++++++++------- .../fs/TestRawLocalFileSystemContract.java | 24 +- .../fs/s3a/ITestS3AFileSystemContract.java | 39 +-- 3 files changed, 178 insertions(+), 132 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java index 62479591826..78ba1f928a1 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java @@ -24,8 +24,9 @@ import java.util.ArrayList; import junit.framework.TestCase; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.security.AccessControlException; @@ -45,8 +46,8 @@ import org.apache.hadoop.util.StringUtils; *

    */ public abstract class FileSystemContractBaseTest extends TestCase { - private static final Log LOG = - LogFactory.getLog(FileSystemContractBaseTest.class); + private static final Logger LOG = + LoggerFactory.getLogger(FileSystemContractBaseTest.class); protected final static String TEST_UMASK = "062"; protected FileSystem fs; @@ -54,15 +55,46 @@ public abstract class FileSystemContractBaseTest extends TestCase { @Override protected void tearDown() throws Exception { - try { - if (fs != null) { - fs.delete(path("/test"), true); + if (fs != null) { + // some cases use this absolute path + if (rootDirTestEnabled()) { + cleanupDir(path("/FileSystemContractBaseTest")); } + // others use this relative path against test base directory + cleanupDir(getTestBaseDir()); + } + super.tearDown(); + } + + private void cleanupDir(Path p) { + try { + LOG.info("Deleting " + p); + fs.delete(p, true); } catch (IOException e) { - LOG.error("Error deleting /test: " + e, e); + LOG.error("Error deleting test dir: " + p, e); } } - + + /** + * Test base directory for resolving relative test paths. + * + * The default value is /user/$USER/FileSystemContractBaseTest. Subclass may + * set specific test base directory. + */ + protected Path getTestBaseDir() { + return new Path(fs.getWorkingDirectory(), "FileSystemContractBaseTest"); + } + + /** + * For absolute path return the fully qualified path while for relative path + * return the fully qualified path against {@link #getTestBaseDir()}. + */ + protected final Path path(String pathString) { + Path p = new Path(pathString).makeQualified(fs.getUri(), getTestBaseDir()); + LOG.info("Resolving {} -> {}", pathString, p); + return p; + } + protected int getBlockSize() { return 1024; } @@ -80,6 +112,17 @@ public abstract class FileSystemContractBaseTest extends TestCase { return true; } + /** + * Override this if the filesystem does not enable testing root directories. + * + * If this returns true, the test will create and delete test directories and + * files under root directory, which may have side effects, e.g. fail tests + * with PermissionDenied exceptions. + */ + protected boolean rootDirTestEnabled() { + return true; + } + /** * Override this if the filesystem is not case sensitive * @return true if the case detection/preservation tests should run @@ -102,24 +145,24 @@ public abstract class FileSystemContractBaseTest extends TestCase { Path workDir = path(getDefaultWorkingDirectory()); assertEquals(workDir, fs.getWorkingDirectory()); - fs.setWorkingDirectory(path(".")); + fs.setWorkingDirectory(fs.makeQualified(new Path("."))); assertEquals(workDir, fs.getWorkingDirectory()); - fs.setWorkingDirectory(path("..")); + fs.setWorkingDirectory(fs.makeQualified(new Path(".."))); assertEquals(workDir.getParent(), fs.getWorkingDirectory()); - Path relativeDir = path("hadoop"); + Path relativeDir = fs.makeQualified(new Path("testWorkingDirectory")); fs.setWorkingDirectory(relativeDir); assertEquals(relativeDir, fs.getWorkingDirectory()); - Path absoluteDir = path("/test/hadoop"); + Path absoluteDir = path("/FileSystemContractBaseTest/testWorkingDirectory"); fs.setWorkingDirectory(absoluteDir); assertEquals(absoluteDir, fs.getWorkingDirectory()); } public void testMkdirs() throws Exception { - Path testDir = path("/test/hadoop"); + Path testDir = path("testMkdirs"); assertFalse(fs.exists(testDir)); assertFalse(fs.isFile(testDir)); @@ -145,14 +188,15 @@ public abstract class FileSystemContractBaseTest extends TestCase { } public void testMkdirsFailsForSubdirectoryOfExistingFile() throws Exception { - Path testDir = path("/test/hadoop"); + Path testDir = path("testMkdirsFailsForSubdirectoryOfExistingFile"); assertFalse(fs.exists(testDir)); assertTrue(fs.mkdirs(testDir)); assertTrue(fs.exists(testDir)); - createFile(path("/test/hadoop/file")); + createFile(path("testMkdirsFailsForSubdirectoryOfExistingFile/file")); - Path testSubDir = path("/test/hadoop/file/subdir"); + Path testSubDir = path( + "testMkdirsFailsForSubdirectoryOfExistingFile/file/subdir"); try { fs.mkdirs(testSubDir); fail("Should throw IOException."); @@ -167,7 +211,8 @@ public abstract class FileSystemContractBaseTest extends TestCase { // file missing execute permission. } - Path testDeepSubDir = path("/test/hadoop/file/deep/sub/dir"); + Path testDeepSubDir = path( + "testMkdirsFailsForSubdirectoryOfExistingFile/file/deep/sub/dir"); try { fs.mkdirs(testDeepSubDir); fail("Should throw IOException."); @@ -190,7 +235,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { String oldUmask = conf.get(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY); try { conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, TEST_UMASK); - final Path dir = path("/test/newDir"); + final Path dir = path("newDir"); assertTrue(fs.mkdirs(dir, new FsPermission((short) 0777))); FileStatus status = fs.getFileStatus(dir); assertTrue(status.isDirectory()); @@ -223,7 +268,8 @@ public abstract class FileSystemContractBaseTest extends TestCase { public void testGetFileStatusThrowsExceptionForNonExistentFile() throws Exception { try { - fs.getFileStatus(path("/test/hadoop/file")); + fs.getFileStatus( + path("testGetFileStatusThrowsExceptionForNonExistentFile/file")); fail("Should throw FileNotFoundException"); } catch (FileNotFoundException e) { // expected @@ -232,7 +278,8 @@ public abstract class FileSystemContractBaseTest extends TestCase { public void testListStatusThrowsExceptionForNonExistentFile() throws Exception { try { - fs.listStatus(path("/test/hadoop/file")); + fs.listStatus( + path("testListStatusThrowsExceptionForNonExistentFile/file")); fail("Should throw FileNotFoundException"); } catch (FileNotFoundException fnfe) { // expected @@ -240,30 +287,32 @@ public abstract class FileSystemContractBaseTest extends TestCase { } public void testListStatus() throws Exception { - Path[] testDirs = { path("/test/hadoop/a"), - path("/test/hadoop/b"), - path("/test/hadoop/c/1"), }; + final Path[] testDirs = { + path("testListStatus/a"), + path("testListStatus/b"), + path("testListStatus/c/1") + }; assertFalse(fs.exists(testDirs[0])); for (Path path : testDirs) { assertTrue(fs.mkdirs(path)); } - FileStatus[] paths = fs.listStatus(path("/test")); + FileStatus[] paths = fs.listStatus(path(".")); assertEquals(1, paths.length); - assertEquals(path("/test/hadoop"), paths[0].getPath()); + assertEquals(path("testListStatus"), paths[0].getPath()); - paths = fs.listStatus(path("/test/hadoop")); + paths = fs.listStatus(path("testListStatus")); assertEquals(3, paths.length); ArrayList list = new ArrayList(); for (FileStatus fileState : paths) { list.add(fileState.getPath()); } - assertTrue(list.contains(path("/test/hadoop/a"))); - assertTrue(list.contains(path("/test/hadoop/b"))); - assertTrue(list.contains(path("/test/hadoop/c"))); + assertTrue(list.contains(path("testListStatus/a"))); + assertTrue(list.contains(path("testListStatus/b"))); + assertTrue(list.contains(path("testListStatus/c"))); - paths = fs.listStatus(path("/test/hadoop/a")); + paths = fs.listStatus(path("testListStatus/a")); assertEquals(0, paths.length); } @@ -294,12 +343,12 @@ public abstract class FileSystemContractBaseTest extends TestCase { * @throws IOException on IO failures */ protected void writeReadAndDelete(int len) throws IOException { - Path path = path("/test/hadoop/file"); + Path path = path("writeReadAndDelete/file"); writeAndRead(path, data, len, false, true); } public void testOverwrite() throws IOException { - Path path = path("/test/hadoop/file"); + Path path = path("testOverwrite/file"); fs.mkdirs(path.getParent()); @@ -325,7 +374,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { } public void testWriteInNonExistentDirectory() throws IOException { - Path path = path("/test/hadoop/file"); + Path path = path("testWriteInNonExistentDirectory/file"); assertFalse("Parent exists", fs.exists(path.getParent())); createFile(path); @@ -335,15 +384,15 @@ public abstract class FileSystemContractBaseTest extends TestCase { } public void testDeleteNonExistentFile() throws IOException { - Path path = path("/test/hadoop/file"); + Path path = path("testDeleteNonExistentFile/file"); assertFalse("Path exists: " + path, fs.exists(path)); assertFalse("No deletion", fs.delete(path, true)); } public void testDeleteRecursively() throws IOException { - Path dir = path("/test/hadoop"); - Path file = path("/test/hadoop/file"); - Path subdir = path("/test/hadoop/subdir"); + Path dir = path("testDeleteRecursively"); + Path file = path("testDeleteRecursively/file"); + Path subdir = path("testDeleteRecursively/subdir"); createFile(file); assertTrue("Created subdir", fs.mkdirs(subdir)); @@ -369,7 +418,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { } public void testDeleteEmptyDirectory() throws IOException { - Path dir = path("/test/hadoop"); + Path dir = path("testDeleteEmptyDirectory"); assertTrue(fs.mkdirs(dir)); assertTrue("Dir exists", fs.exists(dir)); assertTrue("Deleted", fs.delete(dir, false)); @@ -379,26 +428,26 @@ public abstract class FileSystemContractBaseTest extends TestCase { public void testRenameNonExistentPath() throws Exception { if (!renameSupported()) return; - Path src = path("/test/hadoop/path"); - Path dst = path("/test/new/newpath"); + Path src = path("testRenameNonExistentPath/path"); + Path dst = path("testRenameNonExistentPathNew/newpath"); rename(src, dst, false, false, false); } public void testRenameFileMoveToNonExistentDirectory() throws Exception { if (!renameSupported()) return; - Path src = path("/test/hadoop/file"); + Path src = path("testRenameFileMoveToNonExistentDirectory/file"); createFile(src); - Path dst = path("/test/new/newfile"); + Path dst = path("testRenameFileMoveToNonExistentDirectoryNew/newfile"); rename(src, dst, false, true, false); } public void testRenameFileMoveToExistingDirectory() throws Exception { if (!renameSupported()) return; - Path src = path("/test/hadoop/file"); + Path src = path("testRenameFileMoveToExistingDirectory/file"); createFile(src); - Path dst = path("/test/new/newfile"); + Path dst = path("testRenameFileMoveToExistingDirectoryNew/newfile"); fs.mkdirs(dst.getParent()); rename(src, dst, true, false, true); } @@ -406,9 +455,9 @@ public abstract class FileSystemContractBaseTest extends TestCase { public void testRenameFileAsExistingFile() throws Exception { if (!renameSupported()) return; - Path src = path("/test/hadoop/file"); + Path src = path("testRenameFileAsExistingFile/file"); createFile(src); - Path dst = path("/test/new/newfile"); + Path dst = path("testRenameFileAsExistingFileNew/newfile"); createFile(dst); rename(src, dst, false, true, true); } @@ -416,83 +465,84 @@ public abstract class FileSystemContractBaseTest extends TestCase { public void testRenameFileAsExistingDirectory() throws Exception { if (!renameSupported()) return; - Path src = path("/test/hadoop/file"); + Path src = path("testRenameFileAsExistingDirectory/file"); createFile(src); - Path dst = path("/test/new/newdir"); + Path dst = path("testRenameFileAsExistingDirectoryNew/newdir"); fs.mkdirs(dst); rename(src, dst, true, false, true); - assertIsFile(path("/test/new/newdir/file")); + assertIsFile(path("testRenameFileAsExistingDirectoryNew/newdir/file")); } public void testRenameDirectoryMoveToNonExistentDirectory() throws Exception { if (!renameSupported()) return; - Path src = path("/test/hadoop/dir"); + Path src = path("testRenameDirectoryMoveToNonExistentDirectory/dir"); fs.mkdirs(src); - Path dst = path("/test/new/newdir"); + Path dst = path("testRenameDirectoryMoveToNonExistentDirectoryNew/newdir"); rename(src, dst, false, true, false); } public void testRenameDirectoryMoveToExistingDirectory() throws Exception { if (!renameSupported()) return; + final String testDir = "testRenameDirectoryMoveToExistingDirectory"; - Path src = path("/test/hadoop/dir"); + Path src = path(testDir + "/dir"); fs.mkdirs(src); - createFile(path("/test/hadoop/dir/file1")); - createFile(path("/test/hadoop/dir/subdir/file2")); + createFile(path(testDir + "/dir/file1")); + createFile(path(testDir + "/dir/subdir/file2")); - Path dst = path("/test/new/newdir"); + Path dst = path(testDir + "New/newdir"); fs.mkdirs(dst.getParent()); rename(src, dst, true, false, true); assertFalse("Nested file1 exists", - fs.exists(path("/test/hadoop/dir/file1"))); + fs.exists(path(testDir + "/dir/file1"))); assertFalse("Nested file2 exists", - fs.exists(path("/test/hadoop/dir/subdir/file2"))); + fs.exists(path(testDir + "/dir/subdir/file2"))); assertTrue("Renamed nested file1 exists", - fs.exists(path("/test/new/newdir/file1"))); + fs.exists(path(testDir + "New/newdir/file1"))); assertTrue("Renamed nested exists", - fs.exists(path("/test/new/newdir/subdir/file2"))); + fs.exists(path(testDir + "New/newdir/subdir/file2"))); } public void testRenameDirectoryAsExistingFile() throws Exception { if (!renameSupported()) return; - Path src = path("/test/hadoop/dir"); + Path src = path("testRenameDirectoryAsExistingFile/dir"); fs.mkdirs(src); - Path dst = path("/test/new/newfile"); + Path dst = path("testRenameDirectoryAsExistingFileNew/newfile"); createFile(dst); rename(src, dst, false, true, true); } public void testRenameDirectoryAsExistingDirectory() throws Exception { if (!renameSupported()) return; - - Path src = path("/test/hadoop/dir"); + final String testDir = "testRenameDirectoryAsExistingDirectory"; + Path src = path(testDir + "/dir"); fs.mkdirs(src); - createFile(path("/test/hadoop/dir/file1")); - createFile(path("/test/hadoop/dir/subdir/file2")); - - Path dst = path("/test/new/newdir"); + createFile(path(testDir + "/file1")); + createFile(path(testDir + "/subdir/file2")); + + Path dst = path(testDir + "New/newdir"); fs.mkdirs(dst); rename(src, dst, true, false, true); assertTrue("Destination changed", - fs.exists(path("/test/new/newdir/dir"))); + fs.exists(path(testDir + "New/newdir/dir"))); assertFalse("Nested file1 exists", - fs.exists(path("/test/hadoop/dir/file1"))); + fs.exists(path(testDir + "/dir/file1"))); assertFalse("Nested file2 exists", - fs.exists(path("/test/hadoop/dir/subdir/file2"))); + fs.exists(path(testDir + "/dir/subdir/file2"))); assertTrue("Renamed nested file1 exists", - fs.exists(path("/test/new/newdir/dir/file1"))); + fs.exists(path(testDir + "New/newdir/dir/file1"))); assertTrue("Renamed nested exists", - fs.exists(path("/test/new/newdir/dir/subdir/file2"))); + fs.exists(path(testDir + "New/newdir/dir/subdir/file2"))); } public void testInputStreamClosedTwice() throws IOException { //HADOOP-4760 according to Closeable#close() closing already-closed //streams should have no effect. - Path src = path("/test/hadoop/file"); + Path src = path("testInputStreamClosedTwice/file"); createFile(src); FSDataInputStream in = fs.open(src); in.close(); @@ -502,18 +552,13 @@ public abstract class FileSystemContractBaseTest extends TestCase { public void testOutputStreamClosedTwice() throws IOException { //HADOOP-4760 according to Closeable#close() closing already-closed //streams should have no effect. - Path src = path("/test/hadoop/file"); + Path src = path("testOutputStreamClosedTwice/file"); FSDataOutputStream out = fs.create(src); out.writeChar('H'); //write some data out.close(); out.close(); } - - protected Path path(String pathString) { - return new Path(pathString).makeQualified(fs.getUri(), - fs.getWorkingDirectory()); - } - + protected void createFile(Path path) throws IOException { FSDataOutputStream out = fs.create(path); out.write(data, 0, data.length); @@ -541,7 +586,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { byte[] filedata1 = dataset(blockSize * 2, 'A', 26); byte[] filedata2 = dataset(blockSize * 2, 'a', 26); - Path path = path("/test/hadoop/file-overwrite"); + Path path = path("testOverWriteAndRead/file-overwrite"); writeAndRead(path, filedata1, blockSize, true, false); writeAndRead(path, filedata2, blockSize, true, false); writeAndRead(path, filedata1, blockSize * 2, true, false); @@ -561,7 +606,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { LOG.info("Skipping test"); return; } - String mixedCaseFilename = "/test/UPPER.TXT"; + String mixedCaseFilename = "testFilesystemIsCaseSensitive"; Path upper = path(mixedCaseFilename); Path lower = path(StringUtils.toLowerCase(mixedCaseFilename)); assertFalse("File exists" + upper, fs.exists(upper)); @@ -592,7 +637,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { * @throws Exception on failures */ public void testZeroByteFilesAreFiles() throws Exception { - Path src = path("/test/testZeroByteFilesAreFiles"); + Path src = path("testZeroByteFilesAreFiles"); //create a zero byte file FSDataOutputStream out = fs.create(src); out.close(); @@ -605,7 +650,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { * @throws Exception on failures */ public void testMultiByteFilesAreFiles() throws Exception { - Path src = path("/test/testMultiByteFilesAreFiles"); + Path src = path("testMultiByteFilesAreFiles"); FSDataOutputStream out = fs.create(src); out.writeUTF("testMultiByteFilesAreFiles"); out.close(); @@ -629,10 +674,14 @@ public abstract class FileSystemContractBaseTest extends TestCase { * @throws Exception on failures */ public void testRenameRootDirForbidden() throws Exception { + if (!rootDirTestEnabled()) { + return; + } + if (!renameSupported()) return; rename(path("/"), - path("/test/newRootDir"), + path("testRenameRootDirForbidden"), false, true, false); } @@ -644,7 +693,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { public void testRenameChildDirForbidden() throws Exception { if (!renameSupported()) return; LOG.info("testRenameChildDirForbidden"); - Path parentdir = path("/test/parentdir"); + Path parentdir = path("testRenameChildDirForbidden"); fs.mkdirs(parentdir); Path childFile = new Path(parentdir, "childfile"); createFile(childFile); @@ -663,9 +712,9 @@ public abstract class FileSystemContractBaseTest extends TestCase { */ public void testRenameToDirWithSamePrefixAllowed() throws Throwable { if (!renameSupported()) return; - Path parentdir = path("test/parentdir"); + final Path parentdir = path("testRenameToDirWithSamePrefixAllowed"); fs.mkdirs(parentdir); - Path dest = path("test/parentdirdest"); + final Path dest = path("testRenameToDirWithSamePrefixAllowedDest"); rename(parentdir, dest, true, false, true); } @@ -677,7 +726,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { if (!renameSupported()) { return; } - Path parentdir = path("test/parentdir"); + Path parentdir = path("testRenameDirToSelf"); fs.mkdirs(parentdir); Path child = new Path(parentdir, "child"); createFile(child); @@ -696,7 +745,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { if (!renameSupported()) { return; } - Path testdir = path("test/dir"); + Path testdir = path("testMoveDirUnderParent"); fs.mkdirs(testdir); Path parent = testdir.getParent(); //the outcome here is ambiguous, so is not checked @@ -711,7 +760,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { */ public void testRenameFileToSelf() throws Throwable { if (!renameSupported()) return; - Path filepath = path("test/file"); + Path filepath = path("testRenameFileToSelf"); createFile(filepath); //HDFS expects rename src, src -> true rename(filepath, filepath, true, true, true); @@ -725,7 +774,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { */ public void testMoveFileUnderParent() throws Throwable { if (!renameSupported()) return; - Path filepath = path("test/file"); + Path filepath = path("testMoveFileUnderParent"); createFile(filepath); //HDFS expects rename src, src -> true rename(filepath, filepath, true, true, true); @@ -734,15 +783,23 @@ public abstract class FileSystemContractBaseTest extends TestCase { } public void testLSRootDir() throws Throwable { + if (!rootDirTestEnabled()) { + return; + } + Path dir = path("/"); - Path child = path("/test"); + Path child = path("/FileSystemContractBaseTest"); createFile(child); assertListFilesFinds(dir, child); } public void testListStatusRootDir() throws Throwable { + if (!rootDirTestEnabled()) { + return; + } + Path dir = path("/"); - Path child = path("/test"); + Path child = path("/FileSystemContractBaseTest"); createFile(child); assertListStatusFinds(dir, child); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestRawLocalFileSystemContract.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestRawLocalFileSystemContract.java index 036fb6af978..b023c091d3a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestRawLocalFileSystemContract.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestRawLocalFileSystemContract.java @@ -33,6 +33,8 @@ public class TestRawLocalFileSystemContract extends FileSystemContractBaseTest { private static final Logger LOG = LoggerFactory.getLogger(TestRawLocalFileSystemContract.class); + private final static Path TEST_BASE_DIR = + new Path(GenericTestUtils.getTempPath("")); @Before public void setUp() throws Exception { @@ -51,21 +53,25 @@ public class TestRawLocalFileSystemContract extends FileSystemContractBaseTest { return false; } + /** + * Disabling testing root operation. + * + * Writing to root directory on the local file system may get permission + * denied exception, or even worse, delete/overwrite files accidentally. + */ + @Override + protected boolean rootDirTestEnabled() { + return false; + } + @Override public String getDefaultWorkingDirectory() { return fs.getWorkingDirectory().toUri().getPath(); } @Override - protected Path path(String pathString) { - // For testWorkingDirectory - if (pathString.equals(getDefaultWorkingDirectory()) || - pathString.equals(".") || pathString.equals("..")) { - return super.path(pathString); - } - - return new Path(GenericTestUtils.getTempPath(pathString)). - makeQualified(fs.getUri(), fs.getWorkingDirectory()); + protected Path getTestBaseDir() { + return TEST_BASE_DIR; } @Override diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java index f39ec81a79a..5d47e47feb9 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java @@ -57,30 +57,13 @@ public class ITestS3AFileSystemContract extends FileSystemContractBaseTest { fs = S3ATestUtils.createTestFileSystem(conf); basePath = fs.makeQualified( - S3ATestUtils.createTestPath(new Path("/s3afilesystemcontract"))); + S3ATestUtils.createTestPath(new Path("s3afilesystemcontract"))); super.setUp(); } - /** - * This path explicitly places all absolute paths under the per-test suite - * path directory; this allows the test to run in parallel. - * @param pathString path string as input - * @return a qualified path string. - */ - protected Path path(String pathString) { - if (pathString.startsWith("/")) { - return fs.makeQualified(new Path(basePath, pathString)); - } else { - return super.path(pathString); - } - } - @Override - protected void tearDown() throws Exception { - if (fs != null) { - fs.delete(basePath, true); - } - super.tearDown(); + public Path getTestBaseDir() { + return basePath; } @Override @@ -94,22 +77,22 @@ public class ITestS3AFileSystemContract extends FileSystemContractBaseTest { return; } - Path src = path("/test/hadoop/dir"); + Path src = path("testRenameDirectoryAsExisting/dir"); fs.mkdirs(src); - createFile(path("/test/hadoop/dir/file1")); - createFile(path("/test/hadoop/dir/subdir/file2")); + createFile(path("testRenameDirectoryAsExisting/dir/file1")); + createFile(path("testRenameDirectoryAsExisting/dir/subdir/file2")); - Path dst = path("/test/new/newdir"); + Path dst = path("testRenameDirectoryAsExisting/newdir"); fs.mkdirs(dst); rename(src, dst, true, false, true); assertFalse("Nested file1 exists", - fs.exists(path("/test/hadoop/dir/file1"))); + fs.exists(path("testRenameDirectoryAsExisting/dir/file1"))); assertFalse("Nested file2 exists", - fs.exists(path("/test/hadoop/dir/subdir/file2"))); + fs.exists(path("testRenameDirectoryAsExisting/dir/subdir/file2"))); assertTrue("Renamed nested file1 exists", - fs.exists(path("/test/new/newdir/file1"))); + fs.exists(path("testRenameDirectoryAsExisting/newdir/file1"))); assertTrue("Renamed nested exists", - fs.exists(path("/test/new/newdir/subdir/file2"))); + fs.exists(path("testRenameDirectoryAsExisting/newdir/subdir/file2"))); } // @Override From 55796a0946f80a35055701a34379e374399009c5 Mon Sep 17 00:00:00 2001 From: Jing Zhao Date: Mon, 13 Mar 2017 14:14:09 -0700 Subject: [PATCH 062/188] HDFS-11395. RequestHedgingProxyProvider#RequestHedgingInvocationHandler hides the Exception thrown from NameNode. Contributed by Nandakumar. --- .../io/retry/RetryInvocationHandler.java | 17 ++- .../ha/RequestHedgingProxyProvider.java | 41 +++++-- .../ha/TestRequestHedgingProxyProvider.java | 108 +++++++++++++++++- 3 files changed, 151 insertions(+), 15 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java index 8487602910a..ffdd9288c47 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java @@ -240,12 +240,15 @@ public class RetryInvocationHandler implements RpcInvocationHandler { private final long delay; private final RetryAction action; private final long expectedFailoverCount; + private final Exception failException; - RetryInfo(long delay, RetryAction action, long expectedFailoverCount) { + RetryInfo(long delay, RetryAction action, long expectedFailoverCount, + Exception failException) { this.delay = delay; this.retryTime = Time.monotonicNow() + delay; this.action = action; this.expectedFailoverCount = expectedFailoverCount; + this.failException = failException; } boolean isFailover() { @@ -258,11 +261,16 @@ public class RetryInvocationHandler implements RpcInvocationHandler { && action.action == RetryAction.RetryDecision.FAIL; } + Exception getFailException() { + return failException; + } + static RetryInfo newRetryInfo(RetryPolicy policy, Exception e, Counters counters, boolean idempotentOrAtMostOnce, long expectedFailoverCount) throws Exception { RetryAction max = null; long maxRetryDelay = 0; + Exception ex = null; final Iterable exceptions = e instanceof MultiException ? ((MultiException) e).getExceptions().values() @@ -279,10 +287,13 @@ public class RetryInvocationHandler implements RpcInvocationHandler { if (max == null || max.action.compareTo(a.action) < 0) { max = a; + if (a.action == RetryAction.RetryDecision.FAIL) { + ex = exception; + } } } - return new RetryInfo(maxRetryDelay, max, expectedFailoverCount); + return new RetryInfo(maxRetryDelay, max, expectedFailoverCount, ex); } } @@ -359,7 +370,7 @@ public class RetryInvocationHandler implements RpcInvocationHandler { + ". Not retrying because " + retryInfo.action.reason, e); } } - throw e; + throw retryInfo.getFailException(); } log(method, retryInfo.isFailover(), counters.failovers, retryInfo.delay, e); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java index 945e92f68c0..a765e95411d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.hdfs.server.namenode.ha; -import java.io.IOException; import java.lang.reflect.InvocationHandler; import java.lang.reflect.Method; import java.lang.reflect.Proxy; @@ -122,7 +121,7 @@ public class RequestHedgingProxyProvider extends } catch (Exception ex) { ProxyInfo tProxyInfo = proxyMap.get(callResultFuture); logProxyException(ex, tProxyInfo.proxyInfo); - badResults.put(tProxyInfo.proxyInfo, ex); + badResults.put(tProxyInfo.proxyInfo, unwrapException(ex)); LOG.trace("Unsuccessful invocation on [{}]", tProxyInfo.proxyInfo); numAttempts--; } @@ -207,16 +206,36 @@ public class RequestHedgingProxyProvider extends * @return If the exception is caused by an standby namenode. */ private boolean isStandbyException(Exception ex) { - Throwable cause = ex.getCause(); - if (cause != null) { - Throwable cause2 = cause.getCause(); - if (cause2 instanceof RemoteException) { - RemoteException remoteException = (RemoteException)cause2; - IOException unwrapRemoteException = - remoteException.unwrapRemoteException(); - return unwrapRemoteException instanceof StandbyException; - } + Exception exception = unwrapException(ex); + if (exception instanceof RemoteException) { + return ((RemoteException) exception).unwrapRemoteException() + instanceof StandbyException; } return false; } + + /** + * Unwraps the exception.

    + * Example: + * 

    +   * if ex is
+   * ExecutionException(InvocationTargetExeption(SomeException))
+   * returns SomeException
+   *
    + * + * @return unwrapped exception + */ + private Exception unwrapException(Exception ex) { + if (ex != null) { + Throwable cause = ex.getCause(); + if (cause instanceof Exception) { + Throwable innerCause = cause.getCause(); + if (innerCause instanceof Exception) { + return (Exception) innerCause; + } + return (Exception) cause; + } + } + return ex; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java index 3a910c14aa3..37532d5c880 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java @@ -17,7 +17,10 @@ */ package org.apache.hadoop.hdfs.server.namenode.ha; +import java.io.EOFException; +import java.io.FileNotFoundException; import java.io.IOException; +import java.net.ConnectException; import java.net.InetSocketAddress; import java.net.URI; import java.net.URISyntaxException; @@ -30,6 +33,8 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider.ProxyFactory; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; import org.apache.hadoop.io.retry.MultiException; +import org.apache.hadoop.ipc.RemoteException; +import org.apache.hadoop.ipc.StandbyException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Time; @@ -38,6 +43,7 @@ import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import org.mockito.Matchers; import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; @@ -198,7 +204,7 @@ public class TestRequestHedgingProxyProvider { Assert.assertTrue(stats.length == 1); Assert.assertEquals(2, stats[0]); - // Counter shuodl update only once + // Counter should update only once Assert.assertEquals(5, counter.get()); stats = provider.getProxy().proxy.getStats(); @@ -347,6 +353,106 @@ public class TestRequestHedgingProxyProvider { Assert.assertEquals(12, counter.get()); } + @Test + public void testHedgingWhenFileNotFoundException() throws Exception { + NamenodeProtocols active = Mockito.mock(NamenodeProtocols.class); + Mockito + .when(active.getBlockLocations(Matchers.anyString(), + Matchers.anyLong(), Matchers.anyLong())) + .thenThrow(new RemoteException("java.io.FileNotFoundException", + "File does not exist!")); + + NamenodeProtocols standby = Mockito.mock(NamenodeProtocols.class); + Mockito + .when(standby.getBlockLocations(Matchers.anyString(), + Matchers.anyLong(), Matchers.anyLong())) + .thenThrow( + new RemoteException("org.apache.hadoop.ipc.StandbyException", + "Standby NameNode")); + + RequestHedgingProxyProvider provider = + new RequestHedgingProxyProvider<>(conf, nnUri, + NamenodeProtocols.class, createFactory(active, standby)); + try { + provider.getProxy().proxy.getBlockLocations("/tmp/test.file", 0L, 20L); + Assert.fail("Should fail since the active namenode throws" + + " FileNotFoundException!"); + } catch (MultiException me) { + for (Exception ex : me.getExceptions().values()) { + Exception rEx = ((RemoteException) ex).unwrapRemoteException(); + if (rEx instanceof StandbyException) { + continue; + } + Assert.assertTrue(rEx instanceof FileNotFoundException); + } + } + Mockito.verify(active).getBlockLocations(Matchers.anyString(), + Matchers.anyLong(), Matchers.anyLong()); + Mockito.verify(standby).getBlockLocations(Matchers.anyString(), + Matchers.anyLong(), Matchers.anyLong()); + } + + @Test + public void testHedgingWhenConnectException() throws Exception { + NamenodeProtocols active = Mockito.mock(NamenodeProtocols.class); + Mockito.when(active.getStats()).thenThrow(new ConnectException()); + + NamenodeProtocols standby = Mockito.mock(NamenodeProtocols.class); + Mockito.when(standby.getStats()) + .thenThrow( + new RemoteException("org.apache.hadoop.ipc.StandbyException", + "Standby NameNode")); + + RequestHedgingProxyProvider provider = + new RequestHedgingProxyProvider<>(conf, nnUri, + NamenodeProtocols.class, createFactory(active, standby)); + try { + provider.getProxy().proxy.getStats(); + Assert.fail("Should fail since the active namenode throws" + + " ConnectException!"); + } catch (MultiException me) { + for (Exception ex : me.getExceptions().values()) { + if (ex instanceof RemoteException) { + Exception rEx = ((RemoteException) ex) + .unwrapRemoteException(); + Assert.assertTrue("Unexpected RemoteException: " + rEx.getMessage(), + rEx instanceof StandbyException); + } else { + Assert.assertTrue(ex instanceof ConnectException); + } + } + } + Mockito.verify(active).getStats(); + Mockito.verify(standby).getStats(); + } + + @Test + public void testHedgingWhenConnectAndEOFException() throws Exception { + NamenodeProtocols active = Mockito.mock(NamenodeProtocols.class); + Mockito.when(active.getStats()).thenThrow(new EOFException()); + + NamenodeProtocols standby = Mockito.mock(NamenodeProtocols.class); + Mockito.when(standby.getStats()).thenThrow(new ConnectException()); + + RequestHedgingProxyProvider provider = + new RequestHedgingProxyProvider<>(conf, nnUri, + NamenodeProtocols.class, createFactory(active, standby)); + try { + provider.getProxy().proxy.getStats(); + Assert.fail("Should fail since both active and standby namenodes throw" + + " Exceptions!"); + } catch (MultiException me) { + for (Exception ex : me.getExceptions().values()) { + if (!(ex instanceof ConnectException) && + !(ex instanceof EOFException)) { + Assert.fail("Unexpected Exception " + ex.getMessage()); + } + } + } + Mockito.verify(active).getStats(); + Mockito.verify(standby).getStats(); + } + private ProxyFactory createFactory( NamenodeProtocols... protos) { final Iterator iterator = From 9832ae0ed8853d29072c9ea7031cd2373e6b16f9 Mon Sep 17 00:00:00 2001 From: Chen Liang Date: Mon, 13 Mar 2017 17:30:10 -0700 Subject: [PATCH 063/188] HDFS-11482. Add storage type demand to into DFSNetworkTopology#chooseRandom. Contributed by Chen Liang. --- .../org/apache/hadoop/net/InnerNodeImpl.java | 8 +- .../net/NetworkTopologyWithNodeGroup.java | 2 +- .../hadoop/hdfs/net/DFSNetworkTopology.java | 289 ++++++++++++++++++ .../hadoop/hdfs/net}/DFSTopologyNodeImpl.java | 24 +- .../blockmanagement/DatanodeDescriptor.java | 9 + .../hadoop/hdfs/DFSNetworkTopology.java | 36 --- .../{ => net}/TestDFSNetworkTopology.java | 197 +++++++++++- 7 files changed, 519 insertions(+), 46 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java rename hadoop-hdfs-project/hadoop-hdfs/src/{test/java/org/apache/hadoop/hdfs => main/java/org/apache/hadoop/hdfs/net}/DFSTopologyNodeImpl.java (94%) delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSNetworkTopology.java rename hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/{ => net}/TestDFSNetworkTopology.java (60%) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/InnerNodeImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/InnerNodeImpl.java index 81eaf7fdb4c..5a2931bf6a1 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/InnerNodeImpl.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/InnerNodeImpl.java @@ -63,7 +63,7 @@ public class InnerNodeImpl extends NodeBase implements InnerNode { /** Judge if this node represents a rack * @return true if it has no child or its children are not InnerNodes */ - boolean isRack() { + public boolean isRack() { if (children.isEmpty()) { return true; } @@ -81,7 +81,7 @@ public class InnerNodeImpl extends NodeBase implements InnerNode { * @param n a node * @return true if this node is an ancestor of n */ - protected boolean isAncestor(Node n) { + public boolean isAncestor(Node n) { return getPath(this).equals(NodeBase.PATH_SEPARATOR_STR) || (n.getNetworkLocation()+NodeBase.PATH_SEPARATOR_STR). startsWith(getPath(this)+NodeBase.PATH_SEPARATOR_STR); @@ -92,12 +92,12 @@ public class InnerNodeImpl extends NodeBase implements InnerNode { * @param n a node * @return true if this node is the parent of n */ - protected boolean isParent(Node n) { + public boolean isParent(Node n) { return n.getNetworkLocation().equals(getPath(this)); } /* Return a child name of this node who is an ancestor of node n */ - protected String getNextAncestorName(Node n) { + public String getNextAncestorName(Node n) { if (!isAncestor(n)) { throw new IllegalArgumentException( this + "is not an ancestor of " + n); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java index a20d5fc8551..bec0fe13064 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java @@ -308,7 +308,7 @@ public class NetworkTopologyWithNodeGroup extends NetworkTopology { } @Override - boolean isRack() { + public boolean isRack() { // it is node group if (getChildren().isEmpty()) { return false; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java new file mode 100644 index 00000000000..ee83dba0095 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java @@ -0,0 +1,289 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.net; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; +import org.apache.hadoop.net.NetworkTopology; +import org.apache.hadoop.net.Node; +import org.apache.hadoop.net.NodeBase; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Random; + +/** + * The HDFS specific network topology class. The main purpose of doing this + * subclassing is to add storage-type-aware chooseRandom method. All the + * remaining parts should be the same. + * + * Currently a placeholder to test storage type info. + * TODO : add "chooseRandom with storageType info" function. + */ +public class DFSNetworkTopology extends NetworkTopology { + + private static final Random RANDOM = new Random(); + + public static DFSNetworkTopology getInstance(Configuration conf) { + DFSNetworkTopology nt = new DFSNetworkTopology(); + return (DFSNetworkTopology)nt.init(DFSTopologyNodeImpl.FACTORY); + } + + /** + * Randomly choose one node from scope, with specified storage type. + * + * If scope starts with ~, choose one from the all nodes except for the + * ones in scope; otherwise, choose one from scope. + * If excludedNodes is given, choose a node that's not in excludedNodes. + * + * @param scope range of nodes from which a node will be chosen + * @param excludedNodes nodes to be excluded from + * @return the chosen node + */ + public Node chooseRandomWithStorageType(final String scope, + final Collection excludedNodes, StorageType type) { + netlock.readLock().lock(); + try { + if (scope.startsWith("~")) { + return chooseRandomWithStorageType( + NodeBase.ROOT, scope.substring(1), excludedNodes, type); + } else { + return chooseRandomWithStorageType( + scope, null, excludedNodes, type); + } + } finally { + netlock.readLock().unlock(); + } + } + + /** + * Choose a random node based on given scope, excludedScope and excludedNodes + * set. Although in general the topology has at most three layers, this class + * will not impose such assumption. + * + * At high level, the idea is like this, say: + * + * R has two children A and B, and storage type is X, say: + * A has X = 6 (rooted at A there are 6 datanodes with X) and B has X = 8. + * + * Then R will generate a random int between 1~14, if it's <= 6, recursively + * call into A, otherwise B. This will maintain a uniformed randomness of + * choosing datanodes. + * + * The tricky part is how to handle excludes. + * + * For excludedNodes, since this set is small: currently the main reason of + * being an excluded node is because it already has a replica. So randomly + * picking up this node again should be rare. Thus we only check that, if the + * chosen node is excluded, we do chooseRandom again. + * + * For excludedScope, we locate the root of the excluded scope. Subtracting + * all it's ancestors' storage counters accordingly, this way the excluded + * root is out of the picture. + * + * TODO : this function has duplicate code as NetworkTopology, need to + * refactor in the future. + * + * @param scope + * @param excludedScope + * @param excludedNodes + * @return + */ + @VisibleForTesting + Node chooseRandomWithStorageType(final String scope, + String excludedScope, final Collection excludedNodes, + StorageType type) { + if (excludedScope != null) { + if (scope.startsWith(excludedScope)) { + return null; + } + if (!excludedScope.startsWith(scope)) { + excludedScope = null; + } + } + Node node = getNode(scope); + if (node == null) { + LOG.debug("Invalid scope {}, non-existing node", scope); + return null; + } + if (!(node instanceof DFSTopologyNodeImpl)) { + // a node is either DFSTopologyNodeImpl, or a DatanodeDescriptor + return ((DatanodeDescriptor)node).hasStorageType(type) ? node : null; + } + DFSTopologyNodeImpl root = (DFSTopologyNodeImpl)node; + Node excludeRoot = excludedScope == null ? null : getNode(excludedScope); + + // check to see if there are nodes satisfying the condition at all + int availableCount = root.getSubtreeStorageCount(type); + if (excludeRoot != null && root.isAncestor(excludeRoot)) { + if (excludeRoot instanceof DFSTopologyNodeImpl) { + availableCount -= ((DFSTopologyNodeImpl)excludeRoot) + .getSubtreeStorageCount(type); + } else { + availableCount -= ((DatanodeDescriptor)excludeRoot) + .hasStorageType(type) ? 1 : 0; + } + } + if (excludedNodes != null) { + for (Node excludedNode : excludedNodes) { + // all excluded nodes should be DatanodeDescriptor + Preconditions.checkArgument(excludedNode instanceof DatanodeDescriptor); + availableCount -= ((DatanodeDescriptor) excludedNode) + .hasStorageType(type) ? 1 : 0; + } + } + if (availableCount <= 0) { + // should never be <0 in general, adding <0 check for safety purpose + return null; + } + // to this point, it is guaranteed that there is at least one node + // that satisfies the requirement, keep trying until we found one. + Node chosen; + do { + chosen = chooseRandomWithStorageTypeAndExcludeRoot(root, excludeRoot, + type); + if (excludedNodes == null || !excludedNodes.contains(chosen)) { + break; + } else { + LOG.debug("Node {} is excluded, continuing.", chosen); + } + } while (true); + LOG.debug("chooseRandom returning {}", chosen); + return chosen; + } + + /** + * Choose a random node that has the required storage type, under the given + * root, with an excluded subtree root (could also just be a leaf node). + * + * Note that excludedNode is checked after a random node, so it is not being + * handled here. + * + * @param root the root node where we start searching for a datanode + * @param excludeRoot the root of the subtree what should be excluded + * @param type the expected storage type + * @return a random datanode, with the storage type, and is not in excluded + * scope + */ + private Node chooseRandomWithStorageTypeAndExcludeRoot( + DFSTopologyNodeImpl root, Node excludeRoot, StorageType type) { + Node chosenNode; + if (root.isRack()) { + // children are datanode descriptor + ArrayList candidates = new ArrayList<>(); + for (Node node : root.getChildren()) { + if (node.equals(excludeRoot)) { + continue; + } + DatanodeDescriptor dnDescriptor = (DatanodeDescriptor)node; + if (dnDescriptor.hasStorageType(type)) { + candidates.add(node); + } + } + if (candidates.size() == 0) { + return null; + } + // to this point, all nodes in candidates are valid choices, and they are + // all datanodes, pick a random one. + chosenNode = candidates.get(RANDOM.nextInt(candidates.size())); + } else { + // the children are inner nodes + ArrayList candidates = + getEligibleChildren(root, excludeRoot, type); + if (candidates.size() == 0) { + return null; + } + // again, all children are also inner nodes, we can do this cast. + // to maintain uniformality, the search needs to be based on the counts + // of valid datanodes. Below is a random weighted choose. + int totalCounts = 0; + int[] countArray = new int[candidates.size()]; + for (int i = 0; i < candidates.size(); i++) { + DFSTopologyNodeImpl innerNode = candidates.get(i); + int subTreeCount = innerNode.getSubtreeStorageCount(type); + totalCounts += subTreeCount; + countArray[i] = subTreeCount; + } + // generate a random val between [1, totalCounts] + int randomCounts = RANDOM.nextInt(totalCounts) + 1; + int idxChosen = 0; + // searching for the idxChosen can potentially be done with binary + // search, but does not seem to worth it here. + for (int i = 0; i < countArray.length; i++) { + if (randomCounts <= countArray[i]) { + idxChosen = i; + break; + } + randomCounts -= countArray[i]; + } + DFSTopologyNodeImpl nextRoot = candidates.get(idxChosen); + chosenNode = chooseRandomWithStorageTypeAndExcludeRoot( + nextRoot, excludeRoot, type); + } + return chosenNode; + } + + /** + * Given root, excluded root and storage type. Find all the children of the + * root, that has the storage type available. One check is that if the + * excluded root is under a children, this children must subtract the storage + * count of the excluded root. + * @param root the subtree root we check. + * @param excludeRoot the root of the subtree that should be excluded. + * @param type the storage type we look for. + * @return a list of possible nodes, each of them is eligible as the next + * level root we search. + */ + private ArrayList getEligibleChildren( + DFSTopologyNodeImpl root, Node excludeRoot, StorageType type) { + ArrayList candidates = new ArrayList<>(); + int excludeCount = 0; + if (excludeRoot != null && root.isAncestor(excludeRoot)) { + // the subtree to be excluded is under the given root, + // find out the number of nodes to be excluded. + if (excludeRoot instanceof DFSTopologyNodeImpl) { + // if excludedRoot is an inner node, get the counts of all nodes on + // this subtree of that storage type. + excludeCount = ((DFSTopologyNodeImpl) excludeRoot) + .getSubtreeStorageCount(type); + } else { + // if excludedRoot is a datanode, simply ignore this one node + if (((DatanodeDescriptor) excludeRoot).hasStorageType(type)) { + excludeCount = 1; + } + } + } + // have calculated the number of storage counts to be excluded. + // walk through all children to check eligibility. + for (Node node : root.getChildren()) { + DFSTopologyNodeImpl dfsNode = (DFSTopologyNodeImpl) node; + int storageCount = dfsNode.getSubtreeStorageCount(type); + if (excludeRoot != null && excludeCount != 0 && + (dfsNode.isAncestor(excludeRoot) || dfsNode.equals(excludeRoot))) { + storageCount -= excludeCount; + } + if (storageCount > 0) { + candidates.add(dfsNode); + } + } + return candidates; + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTopologyNodeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSTopologyNodeImpl.java similarity index 94% rename from hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTopologyNodeImpl.java rename to hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSTopologyNodeImpl.java index e746823a2ec..92bb30395fc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTopologyNodeImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSTopologyNodeImpl.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.hdfs; +package org.apache.hadoop.hdfs.net; import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.fs.StorageType; @@ -27,6 +27,7 @@ import org.apache.hadoop.net.Node; import java.util.EnumMap; import java.util.EnumSet; import java.util.HashMap; +import java.util.Map; /** * The HDFS-specific representation of a network topology inner node. The @@ -86,6 +87,17 @@ public class DFSTopologyNodeImpl extends InnerNodeImpl { childrenStorageInfo = new HashMap<>(); } + public int getSubtreeStorageCount(StorageType type) { + int res = 0; + for (Map.Entry> entry : + childrenStorageInfo.entrySet()) { + if (entry.getValue().containsKey(type)) { + res += entry.getValue().get(type); + } + } + return res; + } + int getNumOfChildren() { return children.size(); } @@ -178,6 +190,16 @@ public class DFSTopologyNodeImpl extends InnerNodeImpl { parentName, getPath(this), this, this.getLevel() + 1); } + @Override + public boolean equals(Object o) { + return super.equals(o); + } + + @Override + public int hashCode() { + return super.hashCode(); + } + @Override public boolean remove(Node n) { if (!isAncestor(n)) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java index 7b3e4e17d5d..53d272aeca6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java @@ -941,5 +941,14 @@ public class DatanodeDescriptor extends DatanodeInfo { public boolean isRegistered() { return isAlive() && !forceRegistration; } + + public boolean hasStorageType(StorageType type) { + for (DatanodeStorageInfo dnStorage : getStorageInfos()) { + if (dnStorage.getStorageType() == type) { + return true; + } + } + return false; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSNetworkTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSNetworkTopology.java deleted file mode 100644 index a6b8c007abd..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSNetworkTopology.java +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.net.NetworkTopology; - -/** - * The HDFS specific network topology class. The main purpose of doing this - * subclassing is to add storage-type-aware chooseRandom method. All the - * remaining parts should be the same. - * - * Currently a placeholder to test storage type info. - * TODO : add "chooseRandom with storageType info" function. - */ -public class DFSNetworkTopology extends NetworkTopology { - public static DFSNetworkTopology getInstance(Configuration conf) { - DFSNetworkTopology nt = new DFSNetworkTopology(); - return (DFSNetworkTopology)nt.init(DFSTopologyNodeImpl.FACTORY); - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSNetworkTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/net/TestDFSNetworkTopology.java similarity index 60% rename from hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSNetworkTopology.java rename to hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/net/TestDFSNetworkTopology.java index ac1edf9e321..32ecf88670e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSNetworkTopology.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/net/TestDFSNetworkTopology.java @@ -15,14 +15,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.hdfs; +package org.apache.hadoop.hdfs.net; +import com.google.common.collect.Sets; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo; +import org.apache.hadoop.net.Node; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -30,6 +33,8 @@ import org.junit.rules.Timeout; import java.util.EnumMap; import java.util.HashMap; +import java.util.HashSet; +import java.util.Set; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; @@ -62,9 +67,9 @@ public class TestDFSNetworkTopology { "/l2/d4/r1", "/l2/d4/r1"}; final String[] hosts = { "host1", "host2", "host3", "host4", "host5", - "host6", "host7", "host8", "host9", "host10", - "host11", "host12", "host13", "host14", "host15", - "host16", "host17", "host18", "host19", "host20"}; + "host6", "host7", "host8", + "host9", "host10", "host11", "host12", "host13", + "host14", "host15", "host16", "host17", "host18", "host19", "host20"}; final StorageType[] types = { StorageType.ARCHIVE, StorageType.DISK, StorageType.ARCHIVE, StorageType.DISK, StorageType.DISK, @@ -257,4 +262,188 @@ public class TestDFSNetworkTopology { assertNull(CLUSTER.getNode("/l1/d3")); } + + @Test + public void testChooseRandomWithStorageType() throws Exception { + Node n; + DatanodeDescriptor dd; + // test the choose random can return desired storage type nodes without + // exclude + Set diskUnderL1 = + Sets.newHashSet("host2", "host4", "host5", "host6"); + Set archiveUnderL1 = Sets.newHashSet("host1", "host3"); + Set ramdiskUnderL1 = Sets.newHashSet("host7"); + Set ssdUnderL1 = Sets.newHashSet("host8"); + for (int i = 0; i < 10; i++) { + n = CLUSTER.chooseRandomWithStorageType("/l1", null, null, + StorageType.DISK); + assertTrue(n instanceof DatanodeDescriptor); + dd = (DatanodeDescriptor) n; + assertTrue(diskUnderL1.contains(dd.getHostName())); + + n = CLUSTER.chooseRandomWithStorageType("/l1", null, null, + StorageType.RAM_DISK); + assertTrue(n instanceof DatanodeDescriptor); + dd = (DatanodeDescriptor) n; + assertTrue(ramdiskUnderL1.contains(dd.getHostName())); + + n = CLUSTER.chooseRandomWithStorageType("/l1", null, null, + StorageType.ARCHIVE); + assertTrue(n instanceof DatanodeDescriptor); + dd = (DatanodeDescriptor) n; + assertTrue(archiveUnderL1.contains(dd.getHostName())); + + n = CLUSTER.chooseRandomWithStorageType("/l1", null, null, + StorageType.SSD); + assertTrue(n instanceof DatanodeDescriptor); + dd = (DatanodeDescriptor) n; + assertTrue(ssdUnderL1.contains(dd.getHostName())); + } + } + + @Test + public void testChooseRandomWithStorageTypeWithExcluded() throws Exception { + Node n; + DatanodeDescriptor dd; + // below test choose random with exclude, for /l2/d3, every rack has exactly + // one host + // /l2/d3 has five racks r[1~5] but only r4 and r5 have ARCHIVE + // host12 is the one under "/l2/d3/r4", host13 is the one under "/l2/d3/r5" + n = CLUSTER.chooseRandomWithStorageType("/l2/d3/r4", null, null, + StorageType.ARCHIVE); + HashSet excluded = new HashSet<>(); + // exclude the host on r4 (since there is only one host, no randomness here) + excluded.add(n); + + for (int i = 0; i<10; i++) { + n = CLUSTER.chooseRandomWithStorageType("/l2/d3", null, null, + StorageType.ARCHIVE); + assertTrue(n instanceof DatanodeDescriptor); + dd = (DatanodeDescriptor) n; + assertTrue(dd.getHostName().equals("host12") || + dd.getHostName().equals("host13")); + } + + // test exclude nodes + for (int i = 0; i<10; i++) { + n = CLUSTER.chooseRandomWithStorageType("/l2/d3", null, excluded, + StorageType.ARCHIVE); + assertTrue(n instanceof DatanodeDescriptor); + dd = (DatanodeDescriptor) n; + assertTrue(dd.getHostName().equals("host13")); + } + + // test exclude scope + for (int i = 0; i<10; i++) { + n = CLUSTER.chooseRandomWithStorageType("/l2/d3", "/l2/d3/r4", null, + StorageType.ARCHIVE); + assertTrue(n instanceof DatanodeDescriptor); + dd = (DatanodeDescriptor) n; + assertTrue(dd.getHostName().equals("host13")); + } + + // test exclude scope + excluded node with expected null return node + for (int i = 0; i<10; i++) { + n = CLUSTER.chooseRandomWithStorageType("/l2/d3", "/l2/d3/r5", excluded, + StorageType.ARCHIVE); + assertNull(n); + } + + // test exclude scope + excluded node with expected non-null return node + n = CLUSTER.chooseRandomWithStorageType("/l1/d2", null, null, + StorageType.DISK); + dd = (DatanodeDescriptor)n; + assertEquals("host6", dd.getHostName()); + // exclude the host on r4 (since there is only one host, no randomness here) + excluded.add(n); + Set expectedSet = Sets.newHashSet("host4", "host5"); + for (int i = 0; i<10; i++) { + // under l1, there are four hosts with DISK: + // /l1/d1/r1/host2, /l1/d1/r2/host4, /l1/d1/r2/host5 and /l1/d2/r3/host6 + // host6 is excludedNode, host2 is under excluded range scope /l1/d1/r1 + // so should always return r4 or r5 + n = CLUSTER.chooseRandomWithStorageType( + "/l1", "/l1/d1/r1", excluded, StorageType.DISK); + dd = (DatanodeDescriptor) n; + assertTrue(expectedSet.contains(dd.getHostName())); + } + } + + + /** + * This test tests the wrapper method. The wrapper method only takes one scope + * where if it starts with a ~, it is an excluded scope, and searching always + * from root. Otherwise it is a scope. + * @throws Exception throws exception. + */ + @Test + public void testChooseRandomWithStorageTypeWrapper() throws Exception { + Node n; + DatanodeDescriptor dd; + n = CLUSTER.chooseRandomWithStorageType("/l2/d3/r4", null, null, + StorageType.ARCHIVE); + HashSet excluded = new HashSet<>(); + // exclude the host on r4 (since there is only one host, no randomness here) + excluded.add(n); + + // search with given scope being desired scope + for (int i = 0; i<10; i++) { + n = CLUSTER.chooseRandomWithStorageType( + "/l2/d3", null, StorageType.ARCHIVE); + assertTrue(n instanceof DatanodeDescriptor); + dd = (DatanodeDescriptor) n; + assertTrue(dd.getHostName().equals("host12") || + dd.getHostName().equals("host13")); + } + + for (int i = 0; i<10; i++) { + n = CLUSTER.chooseRandomWithStorageType( + "/l2/d3", excluded, StorageType.ARCHIVE); + assertTrue(n instanceof DatanodeDescriptor); + dd = (DatanodeDescriptor) n; + assertTrue(dd.getHostName().equals("host13")); + } + + // search with given scope being exclude scope + + // a total of 4 ramdisk nodes: + // /l1/d2/r3/host7, /l2/d3/r2/host10, /l2/d4/r1/host7 and /l2/d4/r1/host10 + // so if we exclude /l2/d4/r1, if should be always either host7 or host10 + for (int i = 0; i<10; i++) { + n = CLUSTER.chooseRandomWithStorageType( + "~/l2/d4", null, StorageType.RAM_DISK); + assertTrue(n instanceof DatanodeDescriptor); + dd = (DatanodeDescriptor) n; + assertTrue(dd.getHostName().equals("host7") || + dd.getHostName().equals("host10")); + } + + // similar to above, except that we also exclude host10 here. so it should + // always be host7 + n = CLUSTER.chooseRandomWithStorageType("/l2/d3/r2", null, null, + StorageType.RAM_DISK); + // add host10 to exclude + excluded.add(n); + for (int i = 0; i<10; i++) { + n = CLUSTER.chooseRandomWithStorageType( + "~/l2/d4", excluded, StorageType.RAM_DISK); + assertTrue(n instanceof DatanodeDescriptor); + dd = (DatanodeDescriptor) n; + assertTrue(dd.getHostName().equals("host7")); + } + } + + @Test + public void testNonExistingNode() throws Exception { + Node n; + n = CLUSTER.chooseRandomWithStorageType( + "/l100", null, null, StorageType.DISK); + assertNull(n); + n = CLUSTER.chooseRandomWithStorageType( + "/l100/d100", null, null, StorageType.DISK); + assertNull(n); + n = CLUSTER.chooseRandomWithStorageType( + "/l100/d100/r100", null, null, StorageType.DISK); + assertNull(n); + } } From e5f2eedcbfcbfe8fa6fdb6a57b1250f80b12c32f Mon Sep 17 00:00:00 2001 From: Sunil G Date: Tue, 14 Mar 2017 11:47:11 +0530 Subject: [PATCH 064/188] =?UTF-8?q?YARN-5496.=20Make=20Node=20Heatmap=20Ch?= =?UTF-8?q?art=20categories=20clickable=20in=20new=20YARN=20UI.=20Contribu?= =?UTF-8?q?ted=20by=20Gergely=20Nov=C3=A1k.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../app/components/base-chart-component.js | 4 + .../webapp/app/components/nodes-heatmap.js | 106 +++++++++++++----- .../src/main/webapp/app/styles/app.css | 12 ++ 3 files changed, 93 insertions(+), 29 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/base-chart-component.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/base-chart-component.js index d11a532e927..aa418938a7e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/base-chart-component.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/base-chart-component.js @@ -141,4 +141,8 @@ export default Ember.Component.extend({ }; return layout; }, + + willDestroy: function() { + this.tooltip.remove(); + } }); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/nodes-heatmap.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/nodes-heatmap.js index 56528342a82..ef6e46e1fbe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/nodes-heatmap.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/nodes-heatmap.js @@ -26,17 +26,18 @@ export default BaseChartComponent.extend({ CELL_MARGIN: 2, RACK_MARGIN: 20, filter: "", + selectedCategory: 0, - bindTP: function(element) { + bindTP: function(element, cell) { element.on("mouseover", function() { this.tooltip .style("left", (d3.event.pageX) + "px") .style("top", (d3.event.pageY - 28) + "px"); - element.style("opacity", 1.0); + cell.style("opacity", 1.0); }.bind(this)) .on("mousemove", function() { // Handle pie chart case - var text = element.attr("tooltiptext"); + var text = cell.attr("tooltiptext"); this.tooltip.style("opacity", 0.9); this.tooltip.html(text) @@ -45,10 +46,45 @@ export default BaseChartComponent.extend({ }.bind(this)) .on("mouseout", function() { this.tooltip.style("opacity", 0); - element.style("opacity", 0.8); + cell.style("opacity", 0.8); }.bind(this)); }, + bindSelectCategory: function(element, i) { + element.on("click", function() { + if (this.selectedCategory == i) { + // Remove selection for second click + this.selectedCategory = 0; + } else { + this.selectedCategory = i; + } + this.didInsertElement(); + }.bind(this)); + }, + + isNodeSelected: function(node) { + if (this.filter) { + var rack = node.get("rack"); + var host = node.get("nodeHostName"); + if (!rack.includes(this.filter) && !host.includes(this.filter)) { + return false; + } + } + + if (this.selectedCategory === 0) { + return true; + } + + var usage = node.get("usedMemoryMB") / + (node.get("usedMemoryMB") + node.get("availMemoryMB")) + var lowerLimit = (this.selectedCategory - 1) * 0.2; + var upperLimit = this.selectedCategory * 0.2; + if (lowerLimit <= usage && usage <= upperLimit) { + return true; + } + return false; + }, + // data: // [{label=label1, value=value1}, ...] // ... @@ -84,20 +120,32 @@ export default BaseChartComponent.extend({ for (i = 1; i <= 5; i++) { var ratio = i * 0.2 - 0.1; - g.append("rect") + var rect = g.append("rect") .attr("x", sampleXOffset) .attr("y", sampleYOffset) - .attr("fill", colorFunc(ratio)) + .attr("fill", this.selectedCategory === i ? "#2ca02c" : colorFunc(ratio)) .attr("width", this.SAMPLE_CELL_WIDTH) - .attr("height", this.SAMPLE_HEIGHT); - g.append("text") + .attr("height", this.SAMPLE_HEIGHT) + .attr("class", "hyperlink"); + this.bindSelectCategory(rect, i); + var text = g.append("text") .text("" + (ratio * 100).toFixed(1) + "% Used") .attr("y", sampleYOffset + this.SAMPLE_HEIGHT / 2 + 5) .attr("x", sampleXOffset + this.SAMPLE_CELL_WIDTH / 2) - .attr("class", "heatmap-cell"); + .attr("class", "heatmap-cell hyperlink"); + this.bindSelectCategory(text, i); sampleXOffset += this.CELL_MARGIN + this.SAMPLE_CELL_WIDTH; } + if (this.selectedCategory != 0) { + var text = g.append("text") + .text("Clear") + .attr("y", sampleYOffset + this.SAMPLE_HEIGHT / 2 + 5) + .attr("x", sampleXOffset + 20) + .attr("class", "heatmap-clear hyperlink"); + this.bindSelectCategory(text, 0); + } + var chartXOffset = -1; for (i = 0; i < racksArray.length; i++) { @@ -118,22 +166,7 @@ export default BaseChartComponent.extend({ var host = data[j].get("nodeHostName"); if (rack === racksArray[i]) { - if (!rack.includes(this.filter) && !host.includes(this.filter)) { - this.addNode(g, xOffset, yOffset, colorFunc, data[j], false); - g.append("text") - .text(host) - .attr("y", yOffset + this.CELL_HEIGHT / 2 + 5) - .attr("x", xOffset + this.CELL_WIDTH / 2) - .attr("class", "heatmap-cell-notselected"); - } else { - this.addNode(g, xOffset, yOffset, colorFunc, data[j], true); - g.append("text") - .text(host) - .attr("y", yOffset + this.CELL_HEIGHT / 2 + 5) - .attr("x", xOffset + this.CELL_WIDTH / 2) - .attr("class", "heatmap-cell"); - } - + this.addNode(g, xOffset, yOffset, colorFunc, data[j]); xOffset += this.CELL_MARGIN + this.CELL_WIDTH; if (xOffset + this.CELL_MARGIN + this.CELL_WIDTH >= layout.x2 - layout.margin) { @@ -162,7 +195,7 @@ export default BaseChartComponent.extend({ this.renderTitleAndBG(g, title, layout, false); }, - addNode: function (g, xOffset, yOffset, colorFunc, data, selected) { + addNode: function (g, xOffset, yOffset, colorFunc, data) { var rect = g.append("rect") .attr("y", yOffset) .attr("x", xOffset) @@ -171,13 +204,27 @@ export default BaseChartComponent.extend({ (data.get("usedMemoryMB") + data.get("availMemoryMB")))) .attr("width", this.CELL_WIDTH) .attr("tooltiptext", data.get("toolTipText")); - if (selected) { + + if (this.isNodeSelected(data)) { rect.style("opacity", 0.8); - this.bindTP(rect); + this.bindTP(rect, rect); } else { rect.style("opacity", 0.8); rect.attr("fill", "DimGray"); } + var node_id = data.get("id"), + node_addr = data.get("nodeHTTPAddress"), + href = `#/yarn-node/${node_id}/${node_addr}`; + var a = g.append("a") + .attr("href", href); + var text = a.append("text") + .text(data.get("nodeHostName")) + .attr("y", yOffset + this.CELL_HEIGHT / 2 + 5) + .attr("x", xOffset + this.CELL_WIDTH / 2) + .attr("class", this.isNodeSelected(data) ? "heatmap-cell" : "heatmap-cell-notselected") + if (this.isNodeSelected(data)) { + this.bindTP(a, rect); + } }, addPlaceholderNode: function(g, xOffset, yOffset) { @@ -202,7 +249,8 @@ export default BaseChartComponent.extend({ actions: { applyFilter: function(event) { this.filter = event.srcElement.value; + this.selectedCategory = 0; this.didInsertElement(); } } -}); \ No newline at end of file +}); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/styles/app.css b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/styles/app.css index 7ac21bcfe68..dca5b920875 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/styles/app.css +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/styles/app.css @@ -63,6 +63,18 @@ text.heatmap-cell { text-align: center; } +.hyperlink { + cursor: pointer; + -webkit-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; +} + +.heatmap-clear { + fill: #337ab7; +} + text.heatmap-cell-notselected { font: 14px sans-serif; font-weight: bold; From 023b941e3b83f32bc785240dbb1bfce11a987941 Mon Sep 17 00:00:00 2001 From: Yiqun Lin Date: Tue, 14 Mar 2017 17:49:48 +0800 Subject: [PATCH 065/188] HDFS-11526. Fix confusing block recovery message. Contributed by Yiqun Lin. --- .../hdfs/server/datanode/BlockRecoveryWorker.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockRecoveryWorker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockRecoveryWorker.java index d39d0505882..792b6af4ac5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockRecoveryWorker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockRecoveryWorker.java @@ -167,9 +167,8 @@ public class BlockRecoveryWorker { return; } catch (IOException e) { ++errorCount; - InterDatanodeProtocol.LOG.warn( - "Failed to obtain replica info for block (=" + block - + ") from datanode (=" + id + ")", e); + InterDatanodeProtocol.LOG.warn("Failed to recover block (block=" + + block + ", datanode=" + id + ")", e); } } @@ -429,9 +428,8 @@ public class BlockRecoveryWorker { + rBlock.getNewGenerationStamp() + " is aborted.", ripE); return; } catch (IOException e) { - InterDatanodeProtocol.LOG.warn( - "Failed to obtain replica info for block (=" + block - + ") from datanode (=" + id + ")", e); + InterDatanodeProtocol.LOG.warn("Failed to recover block (block=" + + block + ", datanode=" + id + ")", e); } } checkLocations(syncBlocks.size()); From 34424e98a618a9fefce800746168be2b72e17de9 Mon Sep 17 00:00:00 2001 From: Junping Du Date: Tue, 14 Mar 2017 02:56:18 -0700 Subject: [PATCH 066/188] YARN-6314. Potential infinite redirection on YARN log redirection web service. Contributed by Xuan Gong. (cherry picked from commit 5a9dda796f0e73060ada794ad5752cc6a237ab2e) --- .../webapp/AHSWebServices.java | 32 +++++++++++++++---- .../webapp/TestAHSWebServices.java | 17 ++++++++++ .../server/webapp/YarnWebServiceParams.java | 1 + .../nodemanager/webapp/NMWebServices.java | 6 +++- .../nodemanager/webapp/TestNMWebServices.java | 6 ++++ 5 files changed, 54 insertions(+), 8 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java index c296aaa6ada..619519934fa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java @@ -28,6 +28,7 @@ import java.util.Set; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.Path; import javax.ws.rs.PathParam; @@ -235,6 +236,8 @@ public class AHSWebServices extends WebServices { * The container ID * @param nmId * The Node Manager NodeId + * @param redirected_from_node + * Whether this is a redirected request from NM * @return * The log file's name and current file size */ @@ -245,7 +248,9 @@ public class AHSWebServices extends WebServices { @Context HttpServletRequest req, @Context HttpServletResponse res, @PathParam(YarnWebServiceParams.CONTAINER_ID) String containerIdStr, - @QueryParam(YarnWebServiceParams.NM_ID) String nmId) { + @QueryParam(YarnWebServiceParams.NM_ID) String nmId, + @QueryParam(YarnWebServiceParams.REDIRECTED_FROM_NODE) + @DefaultValue("false") boolean redirected_from_node) { ContainerId containerId = null; init(res); try { @@ -253,6 +258,7 @@ public class AHSWebServices extends WebServices { } catch (IllegalArgumentException e) { throw new BadRequestException("invalid container id, " + containerIdStr); } + ApplicationId appId = containerId.getApplicationAttemptId() .getApplicationId(); AppInfo appInfo; @@ -297,9 +303,12 @@ public class AHSWebServices extends WebServices { // make sure nodeHttpAddress is not null and not empty. Otherwise, // we would only get log meta for aggregated logs instead of // re-directing the request - if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) { + if (nodeHttpAddress == null || nodeHttpAddress.isEmpty() + || redirected_from_node) { // return log meta for the aggregated logs if exists. // It will also return empty log meta for the local logs. + // If this is the redirect request from NM, we should not + // re-direct the request back. Simply output the aggregated log meta. return getContainerLogMeta(appId, appOwner, null, containerIdStr, true); } @@ -338,6 +347,8 @@ public class AHSWebServices extends WebServices { * the size of the log file * @param nmId * The Node Manager NodeId + * @param redirected_from_node + * Whether this is the redirect request from NM * @return * The contents of the container's log file */ @@ -352,9 +363,11 @@ public class AHSWebServices extends WebServices { @PathParam(YarnWebServiceParams.CONTAINER_LOG_FILE_NAME) String filename, @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_FORMAT) String format, @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_SIZE) String size, - @QueryParam(YarnWebServiceParams.NM_ID) String nmId) { + @QueryParam(YarnWebServiceParams.NM_ID) String nmId, + @QueryParam(YarnWebServiceParams.REDIRECTED_FROM_NODE) + boolean redirected_from_node) { return getLogs(req, res, containerIdStr, filename, format, - size, nmId); + size, nmId, redirected_from_node); } //TODO: YARN-4993: Refactory ContainersLogsBlock, AggregatedLogsBlock and @@ -371,7 +384,9 @@ public class AHSWebServices extends WebServices { @PathParam(YarnWebServiceParams.CONTAINER_LOG_FILE_NAME) String filename, @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_FORMAT) String format, @QueryParam(YarnWebServiceParams.RESPONSE_CONTENT_SIZE) String size, - @QueryParam(YarnWebServiceParams.NM_ID) String nmId) { + @QueryParam(YarnWebServiceParams.NM_ID) String nmId, + @QueryParam(YarnWebServiceParams.REDIRECTED_FROM_NODE) + @DefaultValue("false") boolean redirected_from_node) { init(res); ContainerId containerId; try { @@ -426,8 +441,11 @@ public class AHSWebServices extends WebServices { nodeHttpAddress = containerInfo.getNodeHttpAddress(); // make sure nodeHttpAddress is not null and not empty. Otherwise, // we would only get aggregated logs instead of re-directing the - // request - if (nodeHttpAddress == null || nodeHttpAddress.isEmpty()) { + // request. + // If this is the redirect request from NM, we should not re-direct the + // request back. Simply output the aggregated logs. + if (nodeHttpAddress == null || nodeHttpAddress.isEmpty() + || redirected_from_node) { // output the aggregated logs return sendStreamOutputResponse(appId, appOwner, null, containerIdStr, filename, format, length, true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java index c2cfb3b4389..dc692a59632 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java @@ -771,6 +771,23 @@ public class TestAHSWebServices extends JerseyTestBase { assertTrue(responseText.contains("LogAggregationType: " + ContainerLogAggregationType.LOCAL)); assertTrue(responseText.contains(AHSWebServices.getNoRedirectWarning())); + + // If this is the redirect request, we would not re-direct the request + // back and get the aggregated logs. + String content1 = "Hello." + containerId1; + NodeId nodeId1 = NodeId.fromString(NM_ID); + TestContainerLogsUtils.createContainerLogFileInRemoteFS(conf, fs, + rootLogDir, containerId1, nodeId1, fileName, user, content1, true); + response = r.path("ws").path("v1") + .path("applicationhistory").path("containers") + .path(containerId1.toString()).path("logs").path(fileName) + .queryParam("user.name", user) + .queryParam(YarnWebServiceParams.REDIRECTED_FROM_NODE, "true") + .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); + responseText = response.getEntity(String.class); + assertTrue(responseText.contains(content1)); + assertTrue(responseText.contains("LogAggregationType: " + + ContainerLogAggregationType.AGGREGATED)); } @Test(timeout = 10000) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java index 87e540fc30f..479cc758138 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/YarnWebServiceParams.java @@ -34,4 +34,5 @@ public interface YarnWebServiceParams { String RESPONSE_CONTENT_FORMAT = "format"; String RESPONSE_CONTENT_SIZE = "size"; String NM_ID = "nm.id"; + String REDIRECTED_FROM_NODE = "redirected_from_node"; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index 4e72746f793..04a889fdc5c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -499,7 +499,11 @@ public class NMWebServices { String requestParams = WebAppUtils.removeQueryParams(httpRequest, YarnWebServiceParams.NM_ID); if (requestParams != null && !requestParams.isEmpty()) { - redirectPath.append("?" + requestParams); + redirectPath.append("?" + requestParams + "&" + + YarnWebServiceParams.REDIRECTED_FROM_NODE + "=true"); + } else { + redirectPath.append("?" + YarnWebServiceParams.REDIRECTED_FROM_NODE + + "=true"); } ResponseBuilder res = Response.status( HttpServletResponse.SC_TEMPORARY_REDIRECT); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java index d9fd2899eec..16411715c55 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java @@ -383,6 +383,8 @@ public class TestNMWebServices extends JerseyTestBase { assertTrue(redirectURL.contains(noExistContainerId.toString())); assertTrue(redirectURL.contains("/logs/" + fileName)); assertTrue(redirectURL.contains("user.name=" + "user")); + assertTrue(redirectURL.contains( + YarnWebServiceParams.REDIRECTED_FROM_NODE + "=true")); assertFalse(redirectURL.contains(YarnWebServiceParams.NM_ID)); // check the new api @@ -397,6 +399,8 @@ public class TestNMWebServices extends JerseyTestBase { assertTrue(redirectURL.contains(noExistContainerId.toString())); assertTrue(redirectURL.contains("/logs/" + fileName)); assertTrue(redirectURL.contains("user.name=" + "user")); + assertTrue(redirectURL.contains( + YarnWebServiceParams.REDIRECTED_FROM_NODE + "=true")); assertFalse(redirectURL.contains(YarnWebServiceParams.NM_ID)); requestURI = r.path("ws").path("v1").path("node") @@ -409,6 +413,8 @@ public class TestNMWebServices extends JerseyTestBase { assertTrue(redirectURL.contains(LOGSERVICEWSADDR)); assertTrue(redirectURL.contains(noExistContainerId.toString())); assertTrue(redirectURL.contains("user.name=" + "user")); + assertTrue(redirectURL.contains( + YarnWebServiceParams.REDIRECTED_FROM_NODE + "=true")); assertFalse(redirectURL.contains(YarnWebServiceParams.NM_ID)); } From 7515e75103c06ce7139b305dd04d4fb2e94b12ad Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Tue, 14 Mar 2017 11:47:25 -0700 Subject: [PATCH 067/188] HDFS-11505. Do not enable any erasure coding policies by default. Contributed by Manoj Govindassamy. --- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 +- .../namenode/ErasureCodingPolicyManager.java | 5 ++++- .../src/main/resources/hdfs-default.xml | 5 +++-- .../src/site/markdown/HDFSErasureCoding.md | 13 +++++++++---- .../hadoop/hdfs/TestDecommissionWithStriped.java | 2 ++ .../hdfs/TestErasureCodeBenchmarkThroughput.java | 2 ++ .../hdfs/TestErasureCodingPolicyWithSnapshot.java | 2 ++ .../org/apache/hadoop/hdfs/TestFileChecksum.java | 2 ++ .../hadoop/hdfs/TestFileStatusWithECPolicy.java | 5 ++++- .../hadoop/hdfs/TestLeaseRecoveryStriped.java | 2 ++ .../hdfs/TestReadStripedFileWithDecoding.java | 2 ++ .../TestReadStripedFileWithMissingBlocks.java | 2 ++ .../hadoop/hdfs/TestReconstructStripedFile.java | 2 ++ .../hadoop/hdfs/TestSafeModeWithStripedFile.java | 2 ++ .../hadoop/hdfs/TestWriteReadStripedFile.java | 2 ++ .../hadoop/hdfs/server/balancer/TestBalancer.java | 2 ++ .../TestBlockTokenWithDFSStriped.java | 4 ++++ ...ReconstructStripedBlocksWithRackAwareness.java | 6 ++++++ .../TestSequentialBlockGroupId.java | 2 ++ .../TestDataNodeErasureCodingMetrics.java | 2 ++ .../hadoop/hdfs/server/mover/TestMover.java | 2 ++ .../TestAddOverReplicatedStripedBlocks.java | 4 +++- .../server/namenode/TestAddStripedBlockInFBR.java | 3 +++ .../server/namenode/TestAddStripedBlocks.java | 10 ++++++---- .../server/namenode/TestEnabledECPolicies.java | 15 +++++++++++++-- .../hdfs/server/namenode/TestFSEditLogLoader.java | 4 ++++ .../hadoop/hdfs/server/namenode/TestFsck.java | 8 ++++++++ .../hdfs/server/namenode/TestNameNodeMXBean.java | 2 ++ .../namenode/TestQuotaWithStripedBlocks.java | 2 ++ .../namenode/TestReconstructStripedBlocks.java | 6 ++++++ .../server/namenode/TestStripedINodeFile.java | 4 ++++ .../TestOfflineImageViewerWithStripedBlocks.java | 2 ++ 32 files changed, 112 insertions(+), 16 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 3fc4980648d..06b33f977d0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -563,7 +563,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys { "10m"; public static final String DFS_NAMENODE_EC_POLICIES_ENABLED_KEY = "dfs.namenode.ec.policies.enabled"; - public static final String DFS_NAMENODE_EC_POLICIES_ENABLED_DEFAULT = "RS-6-3-64k"; + public static final String DFS_NAMENODE_EC_POLICIES_ENABLED_DEFAULT = ""; public static final String DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_THREADS_KEY = "dfs.datanode.ec.reconstruction.stripedread.threads"; public static final int DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_THREADS_DEFAULT = 20; public static final String DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_BUFFER_SIZE_KEY = "dfs.datanode.ec.reconstruction.stripedread.buffer.size"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java index 29af2072995..c23b034ac90 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java @@ -98,12 +98,15 @@ public final class ErasureCodingPolicyManager { DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_DEFAULT); this.enabledPoliciesByName = new TreeMap<>(); for (String policyName : policyNames) { + if (policyName.trim().isEmpty()) { + continue; + } ErasureCodingPolicy ecPolicy = SYSTEM_POLICIES_BY_NAME.get(policyName); if (ecPolicy == null) { String sysPolicies = Arrays.asList(SYS_POLICIES).stream() .map(ErasureCodingPolicy::getName) .collect(Collectors.joining(", ")); - String msg = String.format("EC policy %s specified at %s is not a " + + String msg = String.format("EC policy '%s' specified at %s is not a " + "valid policy. Please choose from list of available policies: " + "[%s]", policyName, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index fa5d814eaf8..2b1495196fe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -2930,10 +2930,11 @@ dfs.namenode.ec.policies.enabled - RS-6-3-64k + Comma-delimited list of enabled erasure coding policies. The NameNode will enforce this when setting an erasure coding policy - on a directory. + on a directory. By default, none of the built-in erasure coding + policies are enabled. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md index 9d9f2ce3c6e..04acdce6ea0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md @@ -66,7 +66,7 @@ Architecture Policies are named *codec*-*num data blocks*-*num parity blocks*-*cell size*. Currently, five built-in policies are supported: `RS-3-2-64k`, `RS-6-3-64k`, `RS-10-4-64k`, `RS-LEGACY-6-3-64k`, and `XOR-2-1-64k`. - By default, only `RS-6-3-64k` is enabled. + By default, all built-in erasure coding policies are disabled. Similar to HDFS storage policies, erasure coding policies are set on a directory. When a file is created, it inherits the EC policy of its nearest ancestor directory. @@ -91,15 +91,20 @@ Deployment Network bisection bandwidth is thus very important. For rack fault-tolerance, it is also important to have at least as many racks as the configured EC stripe width. - For the default EC policy of RS (6,3), this means minimally 9 racks, and ideally 10 or 11 to handle planned and unplanned outages. + For EC policy RS (6,3), this means minimally 9 racks, and ideally 10 or 11 to handle planned and unplanned outages. For clusters with fewer racks than the stripe width, HDFS cannot maintain rack fault-tolerance, but will still attempt to spread a striped file across multiple nodes to preserve node-level fault-tolerance. ### Configuration keys - The set of enabled erasure coding policies can be configured on the NameNode via `dfs.namenode.ec.policies.enabled`. This restricts what EC policies can be set by clients. It does not affect the behavior of already set file or directory-level EC policies. + The set of enabled erasure coding policies can be configured on the NameNode via `dfs.namenode.ec.policies.enabled` configuration. This restricts + what EC policies can be set by clients. It does not affect the behavior of already set file or directory-level EC policies. - By default, only the `RS-6-3-64k` policy is enabled. Typically, the cluster administrator will configure the set of enabled policies based on the size of the cluster and the desired fault-tolerance properties. For instance, for a cluster with 9 racks, a policy like `RS-10-4-64k` will not preserve rack-level fault-tolerance, and `RS-6-3-64k` or `RS-3-2-64k` might be more appropriate. If the administrator only cares about node-level fault-tolerance, `RS-10-4-64k` would still be appropriate as long as there are at least 14 DataNodes in the cluster. + By default, all built-in erasure coding policies are disabled. Typically, the cluster administrator will enable set of policies by including them + in the `dfs .namenode.ec.policies.enabled` configuration based on the size of the cluster and the desired fault-tolerance properties. For instance, + for a cluster with 9 racks, a policy like `RS-10-4-64k` will not preserve rack-level fault-tolerance, and `RS-6-3-64k` or `RS-3-2-64k` might + be more appropriate. If the administrator only cares about node-level fault-tolerance, `RS-10-4-64k` would still be appropriate as long as + there are at least 14 DataNodes in the cluster. The codec implementation for Reed-Solomon and XOR can be configured with the following client and DataNode configuration keys: `io.erasurecode.codec.rs.rawcoder` for the default RS codec, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java index a5df85fcb90..bb394feb7cd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java @@ -131,6 +131,8 @@ public class TestDecommissionWithStriped { conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1); conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY, false); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + StripedFileTestUtil.getDefaultECPolicy().getName()); numDNs = dataBlocks + parityBlocks + 2; cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodeBenchmarkThroughput.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodeBenchmarkThroughput.java index 383e53a243c..be962dc4cd8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodeBenchmarkThroughput.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodeBenchmarkThroughput.java @@ -48,6 +48,8 @@ public class TestErasureCodeBenchmarkThroughput { conf = new HdfsConfiguration(); int numDN = ErasureCodeBenchmarkThroughput.getEcPolicy().getNumDataUnits() + ErasureCodeBenchmarkThroughput.getEcPolicy().getNumParityUnits(); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + ErasureCodeBenchmarkThroughput.getEcPolicy().getName()); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDN).build(); cluster.waitActive(); fs = cluster.getFileSystem(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java index 4833c24d2e6..c5651ee89ef 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java @@ -48,6 +48,8 @@ public class TestErasureCodingPolicyWithSnapshot { @Before public void setupCluster() throws IOException { conf = new HdfsConfiguration(); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + sysDefaultPolicy.getName()); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(groupSize).build(); cluster.waitActive(); fs = cluster.getFileSystem(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java index 7f63e18abd7..b804523b3c0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java @@ -77,6 +77,8 @@ public class TestFileChecksum { conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY, false); conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + StripedFileTestUtil.getDefaultECPolicy().getName()); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); Path ecPath = new Path(ecDir); cluster.getFileSystem().mkdir(ecPath, FsPermission.getDirDefault()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java index 3be0e8d585a..f033ab75b91 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java @@ -43,8 +43,11 @@ public class TestFileStatusWithECPolicy { @Before public void before() throws IOException { + HdfsConfiguration conf = new HdfsConfiguration(); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + StripedFileTestUtil.getDefaultECPolicy().getName()); cluster = - new MiniDFSCluster.Builder(new Configuration()).numDataNodes(1).build(); + new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); cluster.waitActive(); fs = cluster.getFileSystem(); client = fs.getClient(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java index 2ba5aede855..86b1aadf6ea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java @@ -88,6 +88,8 @@ public class TestLeaseRecoveryStriped { false); conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1); conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + ecPolicy.getName()); final int numDNs = dataBlocks + parityBlocks; cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); cluster.waitActive(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java index 4c6ad99c885..cb1640c0b6a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java @@ -100,6 +100,8 @@ public class TestReadStripedFileWithDecoding { conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_STREAMS_HARD_LIMIT_KEY, 0); conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY, false); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + StripedFileTestUtil.getDefaultECPolicy().getName()); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); cluster.getFileSystem().getClient().setErasureCodingPolicy("/", StripedFileTestUtil.getDefaultECPolicy().getName()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java index 930986f0764..34cba92ad21 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java @@ -58,6 +58,8 @@ public class TestReadStripedFileWithMissingBlocks { public void setup() throws IOException { conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + ecPolicy.getName()); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); cluster.getFileSystem().getClient().setErasureCodingPolicy( "/", ecPolicy.getName()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java index 88edcb40d82..affb541891f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java @@ -102,6 +102,8 @@ public class TestReconstructStripedFile { CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY, NativeRSRawErasureCoderFactory.class.getCanonicalName()); } + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + StripedFileTestUtil.getDefaultECPolicy().getName()); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(dnNum).build(); cluster.waitActive(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java index 85538ba9161..edecbf27a6a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java @@ -61,6 +61,8 @@ public class TestSafeModeWithStripedFile { conf = new HdfsConfiguration(); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 100); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + StripedFileTestUtil.getDefaultECPolicy().getName()); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); cluster.getFileSystem().getClient().setErasureCodingPolicy("/", StripedFileTestUtil.getDefaultECPolicy().getName()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java index 9aac65b411e..9b14df14c34 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java @@ -76,6 +76,8 @@ public class TestWriteReadStripedFile { conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY, false); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + StripedFileTestUtil.getDefaultECPolicy().getName()); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); fs = cluster.getFileSystem(); fs.mkdirs(new Path("/ec")); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java index d9a8e5b1659..d2e8f4485e2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java @@ -1929,6 +1929,8 @@ public class TestBalancer { for (int i = 0; i < numOfDatanodes; i++) { racks[i] = "/rack" + (i % numOfRacks); } + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + StripedFileTestUtil.getDefaultECPolicy().getName()); cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(numOfDatanodes) .racks(racks) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java index 82c9fde1d5a..54f28053f64 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java @@ -55,6 +55,8 @@ public class TestBlockTokenWithDFSStriped extends TestBlockTokenWithDFS { private Configuration getConf() { Configuration conf = super.getConf(numDNs); conf.setInt("io.bytes.per.checksum", cellSize); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + StripedFileTestUtil.getDefaultECPolicy().getName()); return conf; } @@ -114,6 +116,8 @@ public class TestBlockTokenWithDFSStriped extends TestBlockTokenWithDFS { public void testEnd2End() throws Exception { Configuration conf = new Configuration(); conf.setBoolean(DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, true); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + StripedFileTestUtil.getDefaultECPolicy().getName()); new TestBalancer().integrationTestWithStripedFile(conf); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java index 9b7c7cd81ed..4ecfd50a30d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java @@ -146,6 +146,8 @@ public class TestReconstructStripedBlocksWithRackAwareness { LOG.info("cluster hosts: {}, racks: {}", Arrays.asList(hosts), Arrays.asList(racks)); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + StripedFileTestUtil.getDefaultECPolicy().getName()); cluster = new MiniDFSCluster.Builder(conf).racks(racks).hosts(hosts) .numDataNodes(hosts.length).build(); cluster.waitActive(); @@ -217,6 +219,8 @@ public class TestReconstructStripedBlocksWithRackAwareness { @Test public void testChooseExcessReplicasToDelete() throws Exception { + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + StripedFileTestUtil.getDefaultECPolicy().getName()); cluster = new MiniDFSCluster.Builder(conf).racks(racks).hosts(hosts) .numDataNodes(hosts.length).build(); cluster.waitActive(); @@ -267,6 +271,8 @@ public class TestReconstructStripedBlocksWithRackAwareness { */ @Test public void testReconstructionWithDecommission() throws Exception { + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + StripedFileTestUtil.getDefaultECPolicy().getName()); final String[] rackNames = getRacks(dataBlocks + parityBlocks + 2, dataBlocks); final String[] hostNames = getHosts(dataBlocks + parityBlocks + 2); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java index 3c18112b566..c5066a04a2a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java @@ -81,6 +81,8 @@ public class TestSequentialBlockGroupId { Configuration conf = new HdfsConfiguration(); conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + StripedFileTestUtil.getDefaultECPolicy().getName()); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); cluster.waitActive(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java index d36db233f5f..ee2afbbd8b5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java @@ -72,6 +72,8 @@ public class TestDataNodeErasureCodingMetrics { conf = new Configuration(); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + StripedFileTestUtil.getDefaultECPolicy().getName()); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); cluster.waitActive(); cluster.getFileSystem().getClient().setErasureCodingPolicy("/", diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java index 0fcb2755955..9af61aedba9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java @@ -508,6 +508,8 @@ public class TestMover { capacities[i][j]=capacity; } } + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + StripedFileTestUtil.getDefaultECPolicy().getName()); final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(numOfDatanodes) .storagesPerDatanode(storagesPerDatanode) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java index 6a4cd32a582..ecbf99d8042 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java @@ -76,13 +76,15 @@ public class TestAddOverReplicatedStripedBlocks { conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0); conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1); conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + ecPolicy.getName()); SimulatedFSDataset.setFactory(conf); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); cluster.waitActive(); fs = cluster.getFileSystem(); fs.mkdirs(dirPath); fs.getClient().setErasureCodingPolicy(dirPath.toString(), - StripedFileTestUtil.getDefaultECPolicy().getName()); + ecPolicy.getName()); } @After diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlockInFBR.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlockInFBR.java index a3afa93e3aa..a4f470b34d8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlockInFBR.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlockInFBR.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.namenode; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; @@ -61,6 +62,8 @@ public class TestAddStripedBlockInFBR { @Before public void setup() throws IOException { Configuration conf = new HdfsConfiguration(); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + StripedFileTestUtil.getDefaultECPolicy().getName()); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(groupSize).build(); cluster.waitActive(); dfs = cluster.getFileSystem(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java index 2eb23b9f829..555c2fac815 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.namenode; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSStripedOutputStream; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; @@ -83,12 +84,13 @@ public class TestAddStripedBlocks { @Before public void setup() throws IOException { - cluster = new MiniDFSCluster.Builder(new HdfsConfiguration()) - .numDataNodes(groupSize).build(); + HdfsConfiguration conf = new HdfsConfiguration(); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + ecPolicy.getName()); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(groupSize).build(); cluster.waitActive(); dfs = cluster.getFileSystem(); - dfs.getClient().setErasureCodingPolicy("/", - StripedFileTestUtil.getDefaultECPolicy().getName()); + dfs.getClient().setErasureCodingPolicy("/", ecPolicy.getName()); } @After diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEnabledECPolicies.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEnabledECPolicies.java index 7f18d18d731..4b4b196a06c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEnabledECPolicies.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEnabledECPolicies.java @@ -69,6 +69,15 @@ public class TestEnabledECPolicies { numEnabled, manager.getEnabledPolicies().length); } + @Test + public void testDefaultPolicy() throws Exception { + HdfsConfiguration conf = new HdfsConfiguration(); + String defaultECPolicies = conf.get( + DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_DEFAULT); + expectValidPolicy(defaultECPolicies, 0); + } + @Test public void testInvalid() throws Exception { // Test first with an invalid policy @@ -90,14 +99,16 @@ public class TestEnabledECPolicies { expectValidPolicy(ecPolicyName, 1); expectValidPolicy(ecPolicyName + ", ", 1); expectValidPolicy(",", 0); + expectValidPolicy(", " + ecPolicyName, 1); + expectValidPolicy(" ", 0); + expectValidPolicy(" , ", 0); } @Test public void testGetPolicies() throws Exception { ErasureCodingPolicy[] enabledPolicies; // Enable no policies - enabledPolicies = new ErasureCodingPolicy[] - {SYSTEM_POLICIES[1], SYSTEM_POLICIES[2]}; + enabledPolicies = new ErasureCodingPolicy[] {}; testGetPolicies(enabledPolicies); // Enable one policy diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java index 104727d83a2..4467dc1068e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java @@ -458,6 +458,8 @@ public class TestFSEditLogLoader { public void testAddNewStripedBlock() throws IOException{ // start a cluster Configuration conf = new HdfsConfiguration(); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + testECPolicy.getName()); MiniDFSCluster cluster = null; try { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(9) @@ -531,6 +533,8 @@ public class TestFSEditLogLoader { public void testUpdateStripedBlocks() throws IOException{ // start a cluster Configuration conf = new HdfsConfiguration(); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + testECPolicy.getName()); MiniDFSCluster cluster = null; try { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(9) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java index b94e6108c4e..e061c5cfe06 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java @@ -682,6 +682,8 @@ public class TestFsck { final int numAllUnits = dataBlocks + ecPolicy.getNumParityUnits(); int blockSize = 2 * cellSize; conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + ecPolicy.getName()); cluster = new MiniDFSCluster.Builder(conf).numDataNodes( numAllUnits + 1).build(); String topDir = "/myDir"; @@ -1997,6 +1999,8 @@ public class TestFsck { conf.setLong(DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY, precision); conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + StripedFileTestUtil.getDefaultECPolicy().getName()); int dataBlocks = StripedFileTestUtil.getDefaultECPolicy().getNumDataUnits(); int parityBlocks = StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits(); @@ -2292,6 +2296,8 @@ public class TestFsck { StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits(); int cellSize = StripedFileTestUtil.getDefaultECPolicy().getCellSize(); int totalSize = dataBlocks + parityBlocks; + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + StripedFileTestUtil.getDefaultECPolicy().getName()); cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(totalSize).build(); fs = cluster.getFileSystem(); @@ -2361,6 +2367,8 @@ public class TestFsck { StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits(); int cellSize = StripedFileTestUtil.getDefaultECPolicy().getCellSize(); int totalSize = dataBlocks + parityBlocks; + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + StripedFileTestUtil.getDefaultECPolicy().getName()); cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(totalSize).build(); fs = cluster.getFileSystem(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java index 231f75fc904..ed9ed3a2133 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java @@ -725,6 +725,8 @@ public class TestNameNodeMXBean { DistributedFileSystem fs = null; try { Configuration conf = new HdfsConfiguration(); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + StripedFileTestUtil.getDefaultECPolicy().getName()); int dataBlocks = StripedFileTestUtil.getDefaultECPolicy().getNumDataUnits(); int parityBlocks = StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java index 1e836dc09bf..f97492b7e04 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java @@ -65,6 +65,8 @@ public class TestQuotaWithStripedBlocks { public void setUp() throws IOException { final Configuration conf = new Configuration(); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + ecPolicy.getName()); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(groupSize).build(); cluster.waitActive(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java index 01a8e08bb1a..e9ff2e360b8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java @@ -108,6 +108,8 @@ public class TestReconstructStripedBlocks { throws Exception { Configuration conf = new HdfsConfiguration(); initConf(conf); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + StripedFileTestUtil.getDefaultECPolicy().getName()); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(groupSize + 1) .build(); @@ -195,6 +197,8 @@ public class TestReconstructStripedBlocks { conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1000); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + StripedFileTestUtil.getDefaultECPolicy().getName()); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(groupSize + 2) .build(); try { @@ -260,6 +264,8 @@ public class TestReconstructStripedBlocks { conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1); conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY, false); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + StripedFileTestUtil.getDefaultECPolicy().getName()); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(groupSize + 2) .build(); cluster.waitActive(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java index f1610b1839e..2c6390011a7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java @@ -311,6 +311,8 @@ public class TestStripedINodeFile { final short GROUP_SIZE = (short) (testECPolicy.getNumDataUnits() + testECPolicy.getNumParityUnits()); conf.setInt(DFSConfigKeys.DFS_NAMENODE_MAX_XATTRS_PER_INODE_KEY, 2); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + StripedFileTestUtil.getDefaultECPolicy().getName()); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(GROUP_SIZE) .build(); @@ -386,6 +388,8 @@ public class TestStripedINodeFile { 1L); conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY, false); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + StripedFileTestUtil.getDefaultECPolicy().getName()); // start 10 datanodes int numOfDatanodes = 10; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java index 0bfa0543026..d04ef99d630 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java @@ -61,6 +61,8 @@ public class TestOfflineImageViewerWithStripedBlocks { int numDNs = dataBlocks + parityBlocks + 2; Configuration conf = new Configuration(); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + StripedFileTestUtil.getDefaultECPolicy().getName()); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); cluster.waitActive(); cluster.getFileSystem().getClient().setErasureCodingPolicy("/", From 0a3aa40fe7878c939dbf4e6b43466595159ff930 Mon Sep 17 00:00:00 2001 From: Naganarasimha Date: Wed, 15 Mar 2017 01:22:25 +0530 Subject: [PATCH 068/188] YARN-6327. Removing queues from CapacitySchedulerQueueManager and ParentQueue should be done with iterator. Contributed by Jonathan Hung. --- .../scheduler/capacity/CapacitySchedulerQueueManager.java | 7 +++++-- .../resourcemanager/scheduler/capacity/ParentQueue.java | 6 ++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java index 8cae6c3a0a6..76cb5d6aac6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -312,10 +313,12 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager< existingQueues.put(queueName, queue); } } - for (Map.Entry e : existingQueues.entrySet()) { + for (Iterator> itr = existingQueues.entrySet() + .iterator(); itr.hasNext();) { + Map.Entry e = itr.next(); String queueName = e.getKey(); if (!newQueues.containsKey(queueName)) { - existingQueues.remove(queueName); + itr.remove(); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index 6f82fcc132a..f84b7a42dbf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -333,10 +333,12 @@ public class ParentQueue extends AbstractCSQueue { } // remove the deleted queue in the refreshed xml. - for (Map.Entry e : currentChildQueues.entrySet()) { + for (Iterator> itr = currentChildQueues + .entrySet().iterator(); itr.hasNext();) { + Map.Entry e = itr.next(); String queueName = e.getKey(); if (!newChildQueues.containsKey(queueName)) { - currentChildQueues.remove(queueName); + itr.remove(); } } From 871dc420f8a4f151189c0925e062c64859a8f275 Mon Sep 17 00:00:00 2001 From: Junping Du Date: Tue, 14 Mar 2017 12:56:54 -0700 Subject: [PATCH 069/188] YARN-6313. YARN logs cli should provide logs for a completed container even when application is still running. Contributed by Xuan Gong. (cherry picked from commit b88f5e0f7858d1d89b79dfd325b767c34416052d) --- .../hadoop/yarn/client/cli/LogsCLI.java | 172 +++++++++++++----- .../hadoop/yarn/client/cli/TestLogsCLI.java | 31 ++++ .../yarn/logaggregation/LogCLIHelpers.java | 11 +- 3 files changed, 160 insertions(+), 54 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java index 3cb1c7d82ba..8407b1930d1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java @@ -44,6 +44,7 @@ import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; +import org.apache.commons.math3.util.Pair; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Evolving; @@ -65,6 +66,7 @@ import org.apache.hadoop.yarn.logaggregation.ContainerLogsRequest; import org.apache.hadoop.yarn.logaggregation.LogCLIHelpers; import org.apache.hadoop.yarn.logaggregation.PerContainerLogFileInfo; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; +import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; @@ -409,10 +411,11 @@ public class LogsCLI extends Configured implements Tool { return false; } - private List getContainerLogFiles( + private List> getContainerLogFiles( Configuration conf, String containerIdStr, String nodeHttpAddress) throws IOException { - List logFileInfos = new ArrayList<>(); + List> logFileInfos + = new ArrayList<>(); Client webServiceClient = Client.create(); try { WebResource webResource = webServiceClient @@ -438,16 +441,20 @@ public class LogsCLI extends Configured implements Tool { } for (int i = 0; i < array.length(); i++) { JSONObject log = array.getJSONObject(i); + String aggregateType = log.has("logAggregationType") ? + log.getString("logAggregationType") : "N/A"; Object ob = log.get("containerLogInfo"); if (ob instanceof JSONArray) { JSONArray obArray = (JSONArray)ob; for (int j = 0; j < obArray.length(); j++) { - logFileInfos.add(generatePerContainerLogFileInfoFromJSON( - obArray.getJSONObject(j))); + logFileInfos.add(new Pair( + generatePerContainerLogFileInfoFromJSON( + obArray.getJSONObject(j)), aggregateType)); } } else if (ob instanceof JSONObject) { - logFileInfos.add(generatePerContainerLogFileInfoFromJSON( - (JSONObject)ob)); + logFileInfos.add(new Pair( + generatePerContainerLogFileInfoFromJSON( + (JSONObject)ob), aggregateType)); } } } catch (Exception e) { @@ -542,10 +549,8 @@ public class LogsCLI extends Configured implements Tool { IOUtils.closeQuietly(is); } } - // for the case, we have already uploaded partial logs in HDFS - int result = logCliHelper.dumpAContainerLogsForLogType( - newOptions, false); - if (result == 0 || foundAnyLogs) { + + if (foundAnyLogs) { return 0; } else { return -1; @@ -586,6 +591,19 @@ public class LogsCLI extends Configured implements Tool { newOptions); } + private int printAggregatedContainerLogs(ContainerLogsRequest request, + LogCLIHelpers logCliHelper, boolean useRegex) throws IOException { + return printContainerLogsForFinishedApplication(request, + logCliHelper, useRegex); + } + + private int printAggregatedContainerLogsWithoutNodeId( + ContainerLogsRequest request, LogCLIHelpers logCliHelper, + boolean useRegex) throws IOException { + return printContainerLogsForFinishedApplicationWithoutNodeId(request, + logCliHelper, useRegex); + } + @Private @VisibleForTesting public ContainerReport getContainerReport(String containerIdStr) @@ -723,9 +741,10 @@ public class LogsCLI extends Configured implements Tool { } private int showContainerLogInfo(ContainerLogsRequest request, - LogCLIHelpers logCliHelper) throws IOException, YarnException { + LogCLIHelpers logCliHelper) throws IOException, YarnException, + ClientHandlerException, UniformInterfaceException, JSONException { if (!request.isAppFinished()) { - return printContainerInfoFromRunningApplication(request); + return printContainerInfoFromRunningApplication(request, logCliHelper); } else { return logCliHelper.printAContainerLogMetadata( request, System.out, System.err); @@ -900,7 +919,8 @@ public class LogsCLI extends Configured implements Tool { } private int fetchContainerLogs(ContainerLogsRequest request, - LogCLIHelpers logCliHelper, boolean useRegex) throws IOException { + LogCLIHelpers logCliHelper, boolean useRegex) throws IOException, + ClientHandlerException, UniformInterfaceException, JSONException { int resultCode = 0; String appIdStr = request.getAppId().toString(); String containerIdStr = request.getContainerId(); @@ -941,14 +961,30 @@ public class LogsCLI extends Configured implements Tool { return printContainerLogsForFinishedApplicationWithoutNodeId( request, logCliHelper, useRegex); } else { - System.err.println("Unable to get logs for this container:" - + containerIdStr + "for the application:" + appIdStr - + " with the appOwner: " + appOwner); - System.err.println("The application: " + appIdStr - + " is still running, and we can not get Container report " - + "for the container: " + containerIdStr +". Please try later " - + "or after the application finishes."); - return -1; + nodeHttpAddress = getNodeHttpAddressFromRMWebString(request); + if (nodeHttpAddress != null && !nodeHttpAddress.isEmpty()) { + request.setNodeHttpAddress(nodeHttpAddress); + } else { + // for the case, we have already uploaded partial logs in HDFS + int result = -1; + if (nodeAddress != null && !nodeAddress.isEmpty()) { + result = printAggregatedContainerLogs( + request, logCliHelper, useRegex); + } else { + result = printAggregatedContainerLogsWithoutNodeId( + request, logCliHelper, useRegex); + } + if (result == -1) { + System.err.println("Unable to get logs for this container:" + + containerIdStr + " for the application:" + appIdStr + + " with the appOwner: " + appOwner); + System.err.println("The application: " + appIdStr + + " is still running, and we can not get Container report " + + "for the container: " + containerIdStr +". Please try later " + + "or after the application finishes."); + } + return result; + } } } // If the application is not in the final state, @@ -1144,7 +1180,9 @@ public class LogsCLI extends Configured implements Tool { } private int printContainerInfoFromRunningApplication( - ContainerLogsRequest options) throws YarnException, IOException { + ContainerLogsRequest options, LogCLIHelpers logCliHelper) + throws YarnException, IOException, ClientHandlerException, + UniformInterfaceException, JSONException { String containerIdStr = options.getContainerId(); String nodeIdStr = options.getNodeId(); List reports = @@ -1152,54 +1190,75 @@ public class LogsCLI extends Configured implements Tool { List filteredReports = filterContainersInfo( options, reports); if (filteredReports.isEmpty()) { - StringBuilder sb = new StringBuilder(); - if (containerIdStr != null && !containerIdStr.isEmpty()) { - sb.append("Trying to get container with ContainerId: " - + containerIdStr + "\n"); + // if we specify the containerId as well as NodeAddress + String nodeHttpAddress = null; + if (options.getContainerId() != null + && !options.getContainerId().isEmpty()) { + nodeHttpAddress = getNodeHttpAddressFromRMWebString(options); } - if (nodeIdStr != null && !nodeIdStr.isEmpty()) { - sb.append("Trying to get container from NodeManager: " - + nodeIdStr + "\n"); + if (nodeHttpAddress != null) { + outputContainerLogMeta(options.getContainerId(), options.getNodeId(), + nodeHttpAddress); + return 0; + } else { + int result = logCliHelper.printAContainerLogMetadata( + options, System.out, System.err); + if (result == -1) { + StringBuilder sb = new StringBuilder(); + if (containerIdStr != null && !containerIdStr.isEmpty()) { + sb.append("Trying to get container with ContainerId: " + + containerIdStr + "\n"); + } + if (nodeIdStr != null && !nodeIdStr.isEmpty()) { + sb.append("Trying to get container from NodeManager: " + + nodeIdStr + "\n"); + } + sb.append("Can not find any matched containers for the application: " + + options.getAppId()); + System.err.println(sb.toString()); + } + return result; } - sb.append("Can not find any matched containers for the application: " - + options.getAppId()); - System.err.println(sb.toString()); - return -1; } for (ContainerReport report : filteredReports) { String nodeId = report.getAssignedNode().toString(); String nodeHttpAddress = report.getNodeHttpAddress().replaceFirst( WebAppUtils.getHttpSchemePrefix(getConf()), ""); String containerId = report.getContainerId().toString(); - String containerString = String.format( - LogCLIHelpers.CONTAINER_ON_NODE_PATTERN, containerId, nodeId); - outStream.println(containerString); - outStream.println(StringUtils.repeat("=", containerString.length())); - outStream.printf(LogCLIHelpers.PER_LOG_FILE_INFO_PATTERN, - "LogFile", "LogLength", "LastModificationTime"); - outStream.println(StringUtils.repeat("=", containerString.length())); - List infos = getContainerLogFiles( - getConf(), containerId, nodeHttpAddress); - for (PerContainerLogFileInfo info : infos) { - outStream.printf(LogCLIHelpers.PER_LOG_FILE_INFO_PATTERN, - info.getFileName(), info.getFileSize(), - info.getLastModifiedTime()); - } + outputContainerLogMeta(containerId, nodeId, nodeHttpAddress); } return 0; } + private void outputContainerLogMeta(String containerId, String nodeId, + String nodeHttpAddress) throws IOException { + String containerString = String.format( + LogCLIHelpers.CONTAINER_ON_NODE_PATTERN, containerId, nodeId); + outStream.println(containerString); + outStream.println(StringUtils.repeat("=", containerString.length())); + outStream.printf(LogCLIHelpers.PER_LOG_FILE_INFO_PATTERN, + "LogFile", "LogLength", "LastModificationTime", "LogAggregationType"); + outStream.println(StringUtils.repeat("=", containerString.length() * 2)); + List> infos = getContainerLogFiles( + getConf(), containerId, nodeHttpAddress); + for (Pair info : infos) { + outStream.printf(LogCLIHelpers.PER_LOG_FILE_INFO_PATTERN, + info.getKey().getFileName(), info.getKey().getFileSize(), + info.getKey().getLastModifiedTime(), info.getValue()); + } + } + @VisibleForTesting public Set getMatchedContainerLogFiles(ContainerLogsRequest request, boolean useRegex) throws IOException { // fetch all the log files for the container // filter the log files based on the given -log_files pattern - List allLogFileInfos= + List> allLogFileInfos= getContainerLogFiles(getConf(), request.getContainerId(), request.getNodeHttpAddress()); List fileNames = new ArrayList(); - for (PerContainerLogFileInfo fileInfo : allLogFileInfos) { - fileNames.add(fileInfo.getFileName()); + for (Pair fileInfo : allLogFileInfos) { + fileNames.add(fileInfo.getKey().getFileName()); } return getMatchedLogFiles(request, fileNames, useRegex); @@ -1217,4 +1276,17 @@ public class LogsCLI extends Configured implements Tool { .queryParam("size", Long.toString(request.getBytes())) .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); } + + @VisibleForTesting + public String getNodeHttpAddressFromRMWebString(ContainerLogsRequest request) + throws ClientHandlerException, UniformInterfaceException, JSONException { + if (request.getNodeId() == null || request.getNodeId().isEmpty()) { + return null; + } + JSONObject nodeInfo = YarnWebServiceUtils + .getNodeInfoFromRMWebService(getConf(), request.getNodeId()) + .getJSONObject("node"); + return nodeInfo.has("nodeHTTPAddress") ? + nodeInfo.getString("nodeHTTPAddress") : null; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index ef164a5c27c..05993d5dd9b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -755,6 +755,23 @@ public class TestLogsCLI { Set logTypes1 = capturedRequests.get(1).getLogTypes(); Assert.assertTrue(logTypes0.contains("ALL") && (logTypes0.size() == 1)); Assert.assertTrue(logTypes1.contains("ALL") && (logTypes1.size() == 1)); + + mockYarnClient = createMockYarnClientWithException( + YarnApplicationState.RUNNING, ugi.getShortUserName()); + LogsCLI cli2 = spy(new LogsCLIForTest(mockYarnClient)); + doReturn(0).when(cli2).printContainerLogsFromRunningApplication( + any(Configuration.class), any(ContainerLogsRequest.class), + any(LogCLIHelpers.class), anyBoolean()); + doReturn("123").when(cli2).getNodeHttpAddressFromRMWebString( + any(ContainerLogsRequest.class)); + cli2.setConf(new YarnConfiguration()); + ContainerId containerId100 = ContainerId.newContainerId(appAttemptId, 100); + exitCode = cli2.run(new String[] {"-applicationId", appId.toString(), + "-containerId", containerId100.toString(), "-nodeAddress", "NM:1234"}); + assertTrue(exitCode == 0); + verify(cli2, times(1)).printContainerLogsFromRunningApplication( + any(Configuration.class), logsRequestCaptor.capture(), + any(LogCLIHelpers.class), anyBoolean()); } @Test (timeout = 15000) @@ -1391,6 +1408,20 @@ public class TestLogsCLI { return mockClient; } + private YarnClient createMockYarnClientWithException( + YarnApplicationState appState, String user) + throws YarnException, IOException { + YarnClient mockClient = mock(YarnClient.class); + ApplicationReport mockAppReport = mock(ApplicationReport.class); + doReturn(user).when(mockAppReport).getUser(); + doReturn(appState).when(mockAppReport).getYarnApplicationState(); + doReturn(mockAppReport).when(mockClient).getApplicationReport( + any(ApplicationId.class)); + doThrow(new YarnException()).when(mockClient).getContainerReport( + any(ContainerId.class)); + return mockClient; + } + private YarnClient createMockYarnClientWithException() throws YarnException, IOException { YarnClient mockClient = mock(YarnClient.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java index 90b64edfda5..cf34a1acf0f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java @@ -51,7 +51,7 @@ import com.google.common.annotations.VisibleForTesting; public class LogCLIHelpers implements Configurable { public static final String PER_LOG_FILE_INFO_PATTERN = - "%30s\t%30s\t%30s" + System.getProperty("line.separator"); + "%30s\t%30s\t%30s\t%30s" + System.getProperty("line.separator"); public static final String CONTAINER_ON_NODE_PATTERN = "Container: %s on %s"; @@ -164,6 +164,7 @@ public class LogCLIHelpers implements Configurable { String containerString = String.format(CONTAINER_ON_NODE_PATTERN, containerId, thisNodeFile.getPath().getName()); out.println(containerString); + out.println("LogAggregationType: AGGREGATED"); out.println(StringUtils.repeat("=", containerString.length())); // We have to re-create reader object to reset the stream index // after calling getContainerLogsStream which would move the stream @@ -238,6 +239,7 @@ public class LogCLIHelpers implements Configurable { String containerString = String.format(CONTAINER_ON_NODE_PATTERN, containerId, thisNodeFile.getPath().getName()); out.println(containerString); + out.println("LogAggregationType: AGGREGATED"); out.println(StringUtils.repeat("=", containerString.length())); if (logType == null || logType.isEmpty()) { if (dumpAContainerLogs(containerId, reader, out, @@ -377,6 +379,7 @@ public class LogCLIHelpers implements Configurable { CONTAINER_ON_NODE_PATTERN, key, thisNodeFile.getPath().getName()); out.println(containerString); + out.println("LogAggregationType: AGGREGATED"); out.println(StringUtils.repeat("=", containerString.length())); while (true) { try { @@ -454,12 +457,12 @@ public class LogCLIHelpers implements Configurable { out.println(containerString); out.println(StringUtils.repeat("=", containerString.length())); out.printf(PER_LOG_FILE_INFO_PATTERN, "LogFile", "LogLength", - "LastModificationTime"); - out.println(StringUtils.repeat("=", containerString.length())); + "LastModificationTime", "LogAggregationType"); + out.println(StringUtils.repeat("=", containerString.length() * 2)); for (PerContainerLogFileInfo logMeta : containerLogMeta .getContainerLogMeta()) { out.printf(PER_LOG_FILE_INFO_PATTERN, logMeta.getFileName(), - logMeta.getFileSize(), logMeta.getLastModifiedTime()); + logMeta.getFileSize(), logMeta.getLastModifiedTime(), "AGGREGATED"); } } return 0; From e6cda5819b1f4bbdcb12487260b1e3b787e11879 Mon Sep 17 00:00:00 2001 From: Mingliang Liu Date: Tue, 14 Mar 2017 12:54:46 -0700 Subject: [PATCH 070/188] Revert "HADOOP-14170. FileSystemContractBaseTest is not cleaning up test directory clearly. Contributed by Mingliang Liu" This reverts commit b8c69557b7a23ff9c4c0b2c9d595338a08b873f1. --- .../hadoop/fs/FileSystemContractBaseTest.java | 247 +++++++----------- .../fs/TestRawLocalFileSystemContract.java | 24 +- .../fs/s3a/ITestS3AFileSystemContract.java | 39 ++- 3 files changed, 132 insertions(+), 178 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java index 78ba1f928a1..62479591826 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java @@ -24,9 +24,8 @@ import java.util.ArrayList; import junit.framework.TestCase; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.security.AccessControlException; @@ -46,8 +45,8 @@ import org.apache.hadoop.util.StringUtils; *

    */ public abstract class FileSystemContractBaseTest extends TestCase { - private static final Logger LOG = - LoggerFactory.getLogger(FileSystemContractBaseTest.class); + private static final Log LOG = + LogFactory.getLog(FileSystemContractBaseTest.class); protected final static String TEST_UMASK = "062"; protected FileSystem fs; @@ -55,46 +54,15 @@ public abstract class FileSystemContractBaseTest extends TestCase { @Override protected void tearDown() throws Exception { - if (fs != null) { - // some cases use this absolute path - if (rootDirTestEnabled()) { - cleanupDir(path("/FileSystemContractBaseTest")); - } - // others use this relative path against test base directory - cleanupDir(getTestBaseDir()); - } - super.tearDown(); - } - - private void cleanupDir(Path p) { try { - LOG.info("Deleting " + p); - fs.delete(p, true); + if (fs != null) { + fs.delete(path("/test"), true); + } } catch (IOException e) { - LOG.error("Error deleting test dir: " + p, e); + LOG.error("Error deleting /test: " + e, e); } } - - /** - * Test base directory for resolving relative test paths. - * - * The default value is /user/$USER/FileSystemContractBaseTest. Subclass may - * set specific test base directory. - */ - protected Path getTestBaseDir() { - return new Path(fs.getWorkingDirectory(), "FileSystemContractBaseTest"); - } - - /** - * For absolute path return the fully qualified path while for relative path - * return the fully qualified path against {@link #getTestBaseDir()}. - */ - protected final Path path(String pathString) { - Path p = new Path(pathString).makeQualified(fs.getUri(), getTestBaseDir()); - LOG.info("Resolving {} -> {}", pathString, p); - return p; - } - + protected int getBlockSize() { return 1024; } @@ -112,17 +80,6 @@ public abstract class FileSystemContractBaseTest extends TestCase { return true; } - /** - * Override this if the filesystem does not enable testing root directories. - * - * If this returns true, the test will create and delete test directories and - * files under root directory, which may have side effects, e.g. fail tests - * with PermissionDenied exceptions. - */ - protected boolean rootDirTestEnabled() { - return true; - } - /** * Override this if the filesystem is not case sensitive * @return true if the case detection/preservation tests should run @@ -145,24 +102,24 @@ public abstract class FileSystemContractBaseTest extends TestCase { Path workDir = path(getDefaultWorkingDirectory()); assertEquals(workDir, fs.getWorkingDirectory()); - fs.setWorkingDirectory(fs.makeQualified(new Path("."))); + fs.setWorkingDirectory(path(".")); assertEquals(workDir, fs.getWorkingDirectory()); - fs.setWorkingDirectory(fs.makeQualified(new Path(".."))); + fs.setWorkingDirectory(path("..")); assertEquals(workDir.getParent(), fs.getWorkingDirectory()); - Path relativeDir = fs.makeQualified(new Path("testWorkingDirectory")); + Path relativeDir = path("hadoop"); fs.setWorkingDirectory(relativeDir); assertEquals(relativeDir, fs.getWorkingDirectory()); - Path absoluteDir = path("/FileSystemContractBaseTest/testWorkingDirectory"); + Path absoluteDir = path("/test/hadoop"); fs.setWorkingDirectory(absoluteDir); assertEquals(absoluteDir, fs.getWorkingDirectory()); } public void testMkdirs() throws Exception { - Path testDir = path("testMkdirs"); + Path testDir = path("/test/hadoop"); assertFalse(fs.exists(testDir)); assertFalse(fs.isFile(testDir)); @@ -188,15 +145,14 @@ public abstract class FileSystemContractBaseTest extends TestCase { } public void testMkdirsFailsForSubdirectoryOfExistingFile() throws Exception { - Path testDir = path("testMkdirsFailsForSubdirectoryOfExistingFile"); + Path testDir = path("/test/hadoop"); assertFalse(fs.exists(testDir)); assertTrue(fs.mkdirs(testDir)); assertTrue(fs.exists(testDir)); - createFile(path("testMkdirsFailsForSubdirectoryOfExistingFile/file")); + createFile(path("/test/hadoop/file")); - Path testSubDir = path( - "testMkdirsFailsForSubdirectoryOfExistingFile/file/subdir"); + Path testSubDir = path("/test/hadoop/file/subdir"); try { fs.mkdirs(testSubDir); fail("Should throw IOException."); @@ -211,8 +167,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { // file missing execute permission. } - Path testDeepSubDir = path( - "testMkdirsFailsForSubdirectoryOfExistingFile/file/deep/sub/dir"); + Path testDeepSubDir = path("/test/hadoop/file/deep/sub/dir"); try { fs.mkdirs(testDeepSubDir); fail("Should throw IOException."); @@ -235,7 +190,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { String oldUmask = conf.get(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY); try { conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, TEST_UMASK); - final Path dir = path("newDir"); + final Path dir = path("/test/newDir"); assertTrue(fs.mkdirs(dir, new FsPermission((short) 0777))); FileStatus status = fs.getFileStatus(dir); assertTrue(status.isDirectory()); @@ -268,8 +223,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { public void testGetFileStatusThrowsExceptionForNonExistentFile() throws Exception { try { - fs.getFileStatus( - path("testGetFileStatusThrowsExceptionForNonExistentFile/file")); + fs.getFileStatus(path("/test/hadoop/file")); fail("Should throw FileNotFoundException"); } catch (FileNotFoundException e) { // expected @@ -278,8 +232,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { public void testListStatusThrowsExceptionForNonExistentFile() throws Exception { try { - fs.listStatus( - path("testListStatusThrowsExceptionForNonExistentFile/file")); + fs.listStatus(path("/test/hadoop/file")); fail("Should throw FileNotFoundException"); } catch (FileNotFoundException fnfe) { // expected @@ -287,32 +240,30 @@ public abstract class FileSystemContractBaseTest extends TestCase { } public void testListStatus() throws Exception { - final Path[] testDirs = { - path("testListStatus/a"), - path("testListStatus/b"), - path("testListStatus/c/1") - }; + Path[] testDirs = { path("/test/hadoop/a"), + path("/test/hadoop/b"), + path("/test/hadoop/c/1"), }; assertFalse(fs.exists(testDirs[0])); for (Path path : testDirs) { assertTrue(fs.mkdirs(path)); } - FileStatus[] paths = fs.listStatus(path(".")); + FileStatus[] paths = fs.listStatus(path("/test")); assertEquals(1, paths.length); - assertEquals(path("testListStatus"), paths[0].getPath()); + assertEquals(path("/test/hadoop"), paths[0].getPath()); - paths = fs.listStatus(path("testListStatus")); + paths = fs.listStatus(path("/test/hadoop")); assertEquals(3, paths.length); ArrayList list = new ArrayList(); for (FileStatus fileState : paths) { list.add(fileState.getPath()); } - assertTrue(list.contains(path("testListStatus/a"))); - assertTrue(list.contains(path("testListStatus/b"))); - assertTrue(list.contains(path("testListStatus/c"))); + assertTrue(list.contains(path("/test/hadoop/a"))); + assertTrue(list.contains(path("/test/hadoop/b"))); + assertTrue(list.contains(path("/test/hadoop/c"))); - paths = fs.listStatus(path("testListStatus/a")); + paths = fs.listStatus(path("/test/hadoop/a")); assertEquals(0, paths.length); } @@ -343,12 +294,12 @@ public abstract class FileSystemContractBaseTest extends TestCase { * @throws IOException on IO failures */ protected void writeReadAndDelete(int len) throws IOException { - Path path = path("writeReadAndDelete/file"); + Path path = path("/test/hadoop/file"); writeAndRead(path, data, len, false, true); } public void testOverwrite() throws IOException { - Path path = path("testOverwrite/file"); + Path path = path("/test/hadoop/file"); fs.mkdirs(path.getParent()); @@ -374,7 +325,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { } public void testWriteInNonExistentDirectory() throws IOException { - Path path = path("testWriteInNonExistentDirectory/file"); + Path path = path("/test/hadoop/file"); assertFalse("Parent exists", fs.exists(path.getParent())); createFile(path); @@ -384,15 +335,15 @@ public abstract class FileSystemContractBaseTest extends TestCase { } public void testDeleteNonExistentFile() throws IOException { - Path path = path("testDeleteNonExistentFile/file"); + Path path = path("/test/hadoop/file"); assertFalse("Path exists: " + path, fs.exists(path)); assertFalse("No deletion", fs.delete(path, true)); } public void testDeleteRecursively() throws IOException { - Path dir = path("testDeleteRecursively"); - Path file = path("testDeleteRecursively/file"); - Path subdir = path("testDeleteRecursively/subdir"); + Path dir = path("/test/hadoop"); + Path file = path("/test/hadoop/file"); + Path subdir = path("/test/hadoop/subdir"); createFile(file); assertTrue("Created subdir", fs.mkdirs(subdir)); @@ -418,7 +369,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { } public void testDeleteEmptyDirectory() throws IOException { - Path dir = path("testDeleteEmptyDirectory"); + Path dir = path("/test/hadoop"); assertTrue(fs.mkdirs(dir)); assertTrue("Dir exists", fs.exists(dir)); assertTrue("Deleted", fs.delete(dir, false)); @@ -428,26 +379,26 @@ public abstract class FileSystemContractBaseTest extends TestCase { public void testRenameNonExistentPath() throws Exception { if (!renameSupported()) return; - Path src = path("testRenameNonExistentPath/path"); - Path dst = path("testRenameNonExistentPathNew/newpath"); + Path src = path("/test/hadoop/path"); + Path dst = path("/test/new/newpath"); rename(src, dst, false, false, false); } public void testRenameFileMoveToNonExistentDirectory() throws Exception { if (!renameSupported()) return; - Path src = path("testRenameFileMoveToNonExistentDirectory/file"); + Path src = path("/test/hadoop/file"); createFile(src); - Path dst = path("testRenameFileMoveToNonExistentDirectoryNew/newfile"); + Path dst = path("/test/new/newfile"); rename(src, dst, false, true, false); } public void testRenameFileMoveToExistingDirectory() throws Exception { if (!renameSupported()) return; - Path src = path("testRenameFileMoveToExistingDirectory/file"); + Path src = path("/test/hadoop/file"); createFile(src); - Path dst = path("testRenameFileMoveToExistingDirectoryNew/newfile"); + Path dst = path("/test/new/newfile"); fs.mkdirs(dst.getParent()); rename(src, dst, true, false, true); } @@ -455,9 +406,9 @@ public abstract class FileSystemContractBaseTest extends TestCase { public void testRenameFileAsExistingFile() throws Exception { if (!renameSupported()) return; - Path src = path("testRenameFileAsExistingFile/file"); + Path src = path("/test/hadoop/file"); createFile(src); - Path dst = path("testRenameFileAsExistingFileNew/newfile"); + Path dst = path("/test/new/newfile"); createFile(dst); rename(src, dst, false, true, true); } @@ -465,84 +416,83 @@ public abstract class FileSystemContractBaseTest extends TestCase { public void testRenameFileAsExistingDirectory() throws Exception { if (!renameSupported()) return; - Path src = path("testRenameFileAsExistingDirectory/file"); + Path src = path("/test/hadoop/file"); createFile(src); - Path dst = path("testRenameFileAsExistingDirectoryNew/newdir"); + Path dst = path("/test/new/newdir"); fs.mkdirs(dst); rename(src, dst, true, false, true); - assertIsFile(path("testRenameFileAsExistingDirectoryNew/newdir/file")); + assertIsFile(path("/test/new/newdir/file")); } public void testRenameDirectoryMoveToNonExistentDirectory() throws Exception { if (!renameSupported()) return; - Path src = path("testRenameDirectoryMoveToNonExistentDirectory/dir"); + Path src = path("/test/hadoop/dir"); fs.mkdirs(src); - Path dst = path("testRenameDirectoryMoveToNonExistentDirectoryNew/newdir"); + Path dst = path("/test/new/newdir"); rename(src, dst, false, true, false); } public void testRenameDirectoryMoveToExistingDirectory() throws Exception { if (!renameSupported()) return; - final String testDir = "testRenameDirectoryMoveToExistingDirectory"; - Path src = path(testDir + "/dir"); + Path src = path("/test/hadoop/dir"); fs.mkdirs(src); - createFile(path(testDir + "/dir/file1")); - createFile(path(testDir + "/dir/subdir/file2")); + createFile(path("/test/hadoop/dir/file1")); + createFile(path("/test/hadoop/dir/subdir/file2")); - Path dst = path(testDir + "New/newdir"); + Path dst = path("/test/new/newdir"); fs.mkdirs(dst.getParent()); rename(src, dst, true, false, true); assertFalse("Nested file1 exists", - fs.exists(path(testDir + "/dir/file1"))); + fs.exists(path("/test/hadoop/dir/file1"))); assertFalse("Nested file2 exists", - fs.exists(path(testDir + "/dir/subdir/file2"))); + fs.exists(path("/test/hadoop/dir/subdir/file2"))); assertTrue("Renamed nested file1 exists", - fs.exists(path(testDir + "New/newdir/file1"))); + fs.exists(path("/test/new/newdir/file1"))); assertTrue("Renamed nested exists", - fs.exists(path(testDir + "New/newdir/subdir/file2"))); + fs.exists(path("/test/new/newdir/subdir/file2"))); } public void testRenameDirectoryAsExistingFile() throws Exception { if (!renameSupported()) return; - Path src = path("testRenameDirectoryAsExistingFile/dir"); + Path src = path("/test/hadoop/dir"); fs.mkdirs(src); - Path dst = path("testRenameDirectoryAsExistingFileNew/newfile"); + Path dst = path("/test/new/newfile"); createFile(dst); rename(src, dst, false, true, true); } public void testRenameDirectoryAsExistingDirectory() throws Exception { if (!renameSupported()) return; - final String testDir = "testRenameDirectoryAsExistingDirectory"; - Path src = path(testDir + "/dir"); + + Path src = path("/test/hadoop/dir"); fs.mkdirs(src); - createFile(path(testDir + "/file1")); - createFile(path(testDir + "/subdir/file2")); - - Path dst = path(testDir + "New/newdir"); + createFile(path("/test/hadoop/dir/file1")); + createFile(path("/test/hadoop/dir/subdir/file2")); + + Path dst = path("/test/new/newdir"); fs.mkdirs(dst); rename(src, dst, true, false, true); assertTrue("Destination changed", - fs.exists(path(testDir + "New/newdir/dir"))); + fs.exists(path("/test/new/newdir/dir"))); assertFalse("Nested file1 exists", - fs.exists(path(testDir + "/dir/file1"))); + fs.exists(path("/test/hadoop/dir/file1"))); assertFalse("Nested file2 exists", - fs.exists(path(testDir + "/dir/subdir/file2"))); + fs.exists(path("/test/hadoop/dir/subdir/file2"))); assertTrue("Renamed nested file1 exists", - fs.exists(path(testDir + "New/newdir/dir/file1"))); + fs.exists(path("/test/new/newdir/dir/file1"))); assertTrue("Renamed nested exists", - fs.exists(path(testDir + "New/newdir/dir/subdir/file2"))); + fs.exists(path("/test/new/newdir/dir/subdir/file2"))); } public void testInputStreamClosedTwice() throws IOException { //HADOOP-4760 according to Closeable#close() closing already-closed //streams should have no effect. - Path src = path("testInputStreamClosedTwice/file"); + Path src = path("/test/hadoop/file"); createFile(src); FSDataInputStream in = fs.open(src); in.close(); @@ -552,13 +502,18 @@ public abstract class FileSystemContractBaseTest extends TestCase { public void testOutputStreamClosedTwice() throws IOException { //HADOOP-4760 according to Closeable#close() closing already-closed //streams should have no effect. - Path src = path("testOutputStreamClosedTwice/file"); + Path src = path("/test/hadoop/file"); FSDataOutputStream out = fs.create(src); out.writeChar('H'); //write some data out.close(); out.close(); } - + + protected Path path(String pathString) { + return new Path(pathString).makeQualified(fs.getUri(), + fs.getWorkingDirectory()); + } + protected void createFile(Path path) throws IOException { FSDataOutputStream out = fs.create(path); out.write(data, 0, data.length); @@ -586,7 +541,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { byte[] filedata1 = dataset(blockSize * 2, 'A', 26); byte[] filedata2 = dataset(blockSize * 2, 'a', 26); - Path path = path("testOverWriteAndRead/file-overwrite"); + Path path = path("/test/hadoop/file-overwrite"); writeAndRead(path, filedata1, blockSize, true, false); writeAndRead(path, filedata2, blockSize, true, false); writeAndRead(path, filedata1, blockSize * 2, true, false); @@ -606,7 +561,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { LOG.info("Skipping test"); return; } - String mixedCaseFilename = "testFilesystemIsCaseSensitive"; + String mixedCaseFilename = "/test/UPPER.TXT"; Path upper = path(mixedCaseFilename); Path lower = path(StringUtils.toLowerCase(mixedCaseFilename)); assertFalse("File exists" + upper, fs.exists(upper)); @@ -637,7 +592,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { * @throws Exception on failures */ public void testZeroByteFilesAreFiles() throws Exception { - Path src = path("testZeroByteFilesAreFiles"); + Path src = path("/test/testZeroByteFilesAreFiles"); //create a zero byte file FSDataOutputStream out = fs.create(src); out.close(); @@ -650,7 +605,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { * @throws Exception on failures */ public void testMultiByteFilesAreFiles() throws Exception { - Path src = path("testMultiByteFilesAreFiles"); + Path src = path("/test/testMultiByteFilesAreFiles"); FSDataOutputStream out = fs.create(src); out.writeUTF("testMultiByteFilesAreFiles"); out.close(); @@ -674,14 +629,10 @@ public abstract class FileSystemContractBaseTest extends TestCase { * @throws Exception on failures */ public void testRenameRootDirForbidden() throws Exception { - if (!rootDirTestEnabled()) { - return; - } - if (!renameSupported()) return; rename(path("/"), - path("testRenameRootDirForbidden"), + path("/test/newRootDir"), false, true, false); } @@ -693,7 +644,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { public void testRenameChildDirForbidden() throws Exception { if (!renameSupported()) return; LOG.info("testRenameChildDirForbidden"); - Path parentdir = path("testRenameChildDirForbidden"); + Path parentdir = path("/test/parentdir"); fs.mkdirs(parentdir); Path childFile = new Path(parentdir, "childfile"); createFile(childFile); @@ -712,9 +663,9 @@ public abstract class FileSystemContractBaseTest extends TestCase { */ public void testRenameToDirWithSamePrefixAllowed() throws Throwable { if (!renameSupported()) return; - final Path parentdir = path("testRenameToDirWithSamePrefixAllowed"); + Path parentdir = path("test/parentdir"); fs.mkdirs(parentdir); - final Path dest = path("testRenameToDirWithSamePrefixAllowedDest"); + Path dest = path("test/parentdirdest"); rename(parentdir, dest, true, false, true); } @@ -726,7 +677,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { if (!renameSupported()) { return; } - Path parentdir = path("testRenameDirToSelf"); + Path parentdir = path("test/parentdir"); fs.mkdirs(parentdir); Path child = new Path(parentdir, "child"); createFile(child); @@ -745,7 +696,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { if (!renameSupported()) { return; } - Path testdir = path("testMoveDirUnderParent"); + Path testdir = path("test/dir"); fs.mkdirs(testdir); Path parent = testdir.getParent(); //the outcome here is ambiguous, so is not checked @@ -760,7 +711,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { */ public void testRenameFileToSelf() throws Throwable { if (!renameSupported()) return; - Path filepath = path("testRenameFileToSelf"); + Path filepath = path("test/file"); createFile(filepath); //HDFS expects rename src, src -> true rename(filepath, filepath, true, true, true); @@ -774,7 +725,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { */ public void testMoveFileUnderParent() throws Throwable { if (!renameSupported()) return; - Path filepath = path("testMoveFileUnderParent"); + Path filepath = path("test/file"); createFile(filepath); //HDFS expects rename src, src -> true rename(filepath, filepath, true, true, true); @@ -783,23 +734,15 @@ public abstract class FileSystemContractBaseTest extends TestCase { } public void testLSRootDir() throws Throwable { - if (!rootDirTestEnabled()) { - return; - } - Path dir = path("/"); - Path child = path("/FileSystemContractBaseTest"); + Path child = path("/test"); createFile(child); assertListFilesFinds(dir, child); } public void testListStatusRootDir() throws Throwable { - if (!rootDirTestEnabled()) { - return; - } - Path dir = path("/"); - Path child = path("/FileSystemContractBaseTest"); + Path child = path("/test"); createFile(child); assertListStatusFinds(dir, child); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestRawLocalFileSystemContract.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestRawLocalFileSystemContract.java index b023c091d3a..036fb6af978 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestRawLocalFileSystemContract.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestRawLocalFileSystemContract.java @@ -33,8 +33,6 @@ public class TestRawLocalFileSystemContract extends FileSystemContractBaseTest { private static final Logger LOG = LoggerFactory.getLogger(TestRawLocalFileSystemContract.class); - private final static Path TEST_BASE_DIR = - new Path(GenericTestUtils.getTempPath("")); @Before public void setUp() throws Exception { @@ -53,25 +51,21 @@ public class TestRawLocalFileSystemContract extends FileSystemContractBaseTest { return false; } - /** - * Disabling testing root operation. - * - * Writing to root directory on the local file system may get permission - * denied exception, or even worse, delete/overwrite files accidentally. - */ - @Override - protected boolean rootDirTestEnabled() { - return false; - } - @Override public String getDefaultWorkingDirectory() { return fs.getWorkingDirectory().toUri().getPath(); } @Override - protected Path getTestBaseDir() { - return TEST_BASE_DIR; + protected Path path(String pathString) { + // For testWorkingDirectory + if (pathString.equals(getDefaultWorkingDirectory()) || + pathString.equals(".") || pathString.equals("..")) { + return super.path(pathString); + } + + return new Path(GenericTestUtils.getTempPath(pathString)). + makeQualified(fs.getUri(), fs.getWorkingDirectory()); } @Override diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java index 5d47e47feb9..f39ec81a79a 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java @@ -57,13 +57,30 @@ public class ITestS3AFileSystemContract extends FileSystemContractBaseTest { fs = S3ATestUtils.createTestFileSystem(conf); basePath = fs.makeQualified( - S3ATestUtils.createTestPath(new Path("s3afilesystemcontract"))); + S3ATestUtils.createTestPath(new Path("/s3afilesystemcontract"))); super.setUp(); } + /** + * This path explicitly places all absolute paths under the per-test suite + * path directory; this allows the test to run in parallel. + * @param pathString path string as input + * @return a qualified path string. + */ + protected Path path(String pathString) { + if (pathString.startsWith("/")) { + return fs.makeQualified(new Path(basePath, pathString)); + } else { + return super.path(pathString); + } + } + @Override - public Path getTestBaseDir() { - return basePath; + protected void tearDown() throws Exception { + if (fs != null) { + fs.delete(basePath, true); + } + super.tearDown(); } @Override @@ -77,22 +94,22 @@ public class ITestS3AFileSystemContract extends FileSystemContractBaseTest { return; } - Path src = path("testRenameDirectoryAsExisting/dir"); + Path src = path("/test/hadoop/dir"); fs.mkdirs(src); - createFile(path("testRenameDirectoryAsExisting/dir/file1")); - createFile(path("testRenameDirectoryAsExisting/dir/subdir/file2")); + createFile(path("/test/hadoop/dir/file1")); + createFile(path("/test/hadoop/dir/subdir/file2")); - Path dst = path("testRenameDirectoryAsExisting/newdir"); + Path dst = path("/test/new/newdir"); fs.mkdirs(dst); rename(src, dst, true, false, true); assertFalse("Nested file1 exists", - fs.exists(path("testRenameDirectoryAsExisting/dir/file1"))); + fs.exists(path("/test/hadoop/dir/file1"))); assertFalse("Nested file2 exists", - fs.exists(path("testRenameDirectoryAsExisting/dir/subdir/file2"))); + fs.exists(path("/test/hadoop/dir/subdir/file2"))); assertTrue("Renamed nested file1 exists", - fs.exists(path("testRenameDirectoryAsExisting/newdir/file1"))); + fs.exists(path("/test/new/newdir/file1"))); assertTrue("Renamed nested exists", - fs.exists(path("testRenameDirectoryAsExisting/newdir/subdir/file2"))); + fs.exists(path("/test/new/newdir/subdir/file2"))); } // @Override From fa67a96d7b0812a6557e40a6ef1eb16f19823e73 Mon Sep 17 00:00:00 2001 From: Mingliang Liu Date: Fri, 10 Mar 2017 18:44:27 -0800 Subject: [PATCH 071/188] HADOOP-14170. FileSystemContractBaseTest is not cleaning up test directory clearly. Contributed by Mingliang Liu --- .../hadoop/fs/FileSystemContractBaseTest.java | 246 +++++++++++------- .../fs/TestRawLocalFileSystemContract.java | 24 +- .../fs/s3a/ITestS3AFileSystemContract.java | 39 +-- 3 files changed, 176 insertions(+), 133 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java index 62479591826..040e9c80746 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java @@ -24,8 +24,9 @@ import java.util.ArrayList; import junit.framework.TestCase; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.security.AccessControlException; @@ -45,8 +46,8 @@ import org.apache.hadoop.util.StringUtils; *

    */ public abstract class FileSystemContractBaseTest extends TestCase { - private static final Log LOG = - LogFactory.getLog(FileSystemContractBaseTest.class); + private static final Logger LOG = + LoggerFactory.getLogger(FileSystemContractBaseTest.class); protected final static String TEST_UMASK = "062"; protected FileSystem fs; @@ -54,15 +55,46 @@ public abstract class FileSystemContractBaseTest extends TestCase { @Override protected void tearDown() throws Exception { - try { - if (fs != null) { - fs.delete(path("/test"), true); + if (fs != null) { + // some cases use this absolute path + if (rootDirTestEnabled()) { + cleanupDir(path("/FileSystemContractBaseTest")); } + // others use this relative path against test base directory + cleanupDir(getTestBaseDir()); + } + super.tearDown(); + } + + private void cleanupDir(Path p) { + try { + LOG.info("Deleting " + p); + fs.delete(p, true); } catch (IOException e) { - LOG.error("Error deleting /test: " + e, e); + LOG.error("Error deleting test dir: " + p, e); } } - + + /** + * Test base directory for resolving relative test paths. + * + * The default value is /user/$USER/FileSystemContractBaseTest. Subclass may + * set specific test base directory. + */ + protected Path getTestBaseDir() { + return new Path(fs.getWorkingDirectory(), "FileSystemContractBaseTest"); + } + + /** + * For absolute path return the fully qualified path while for relative path + * return the fully qualified path against {@link #getTestBaseDir()}. + */ + protected final Path path(String pathString) { + Path p = new Path(pathString).makeQualified(fs.getUri(), getTestBaseDir()); + LOG.info("Resolving {} -> {}", pathString, p); + return p; + } + protected int getBlockSize() { return 1024; } @@ -80,6 +112,17 @@ public abstract class FileSystemContractBaseTest extends TestCase { return true; } + /** + * Override this if the filesystem does not enable testing root directories. + * + * If this returns true, the test will create and delete test directories and + * files under root directory, which may have side effects, e.g. fail tests + * with PermissionDenied exceptions. + */ + protected boolean rootDirTestEnabled() { + return true; + } + /** * Override this if the filesystem is not case sensitive * @return true if the case detection/preservation tests should run @@ -102,24 +145,24 @@ public abstract class FileSystemContractBaseTest extends TestCase { Path workDir = path(getDefaultWorkingDirectory()); assertEquals(workDir, fs.getWorkingDirectory()); - fs.setWorkingDirectory(path(".")); + fs.setWorkingDirectory(fs.makeQualified(new Path("."))); assertEquals(workDir, fs.getWorkingDirectory()); - fs.setWorkingDirectory(path("..")); + fs.setWorkingDirectory(fs.makeQualified(new Path(".."))); assertEquals(workDir.getParent(), fs.getWorkingDirectory()); - Path relativeDir = path("hadoop"); + Path relativeDir = fs.makeQualified(new Path("testWorkingDirectory")); fs.setWorkingDirectory(relativeDir); assertEquals(relativeDir, fs.getWorkingDirectory()); - Path absoluteDir = path("/test/hadoop"); + Path absoluteDir = path("/FileSystemContractBaseTest/testWorkingDirectory"); fs.setWorkingDirectory(absoluteDir); assertEquals(absoluteDir, fs.getWorkingDirectory()); } public void testMkdirs() throws Exception { - Path testDir = path("/test/hadoop"); + Path testDir = path("testMkdirs"); assertFalse(fs.exists(testDir)); assertFalse(fs.isFile(testDir)); @@ -145,14 +188,15 @@ public abstract class FileSystemContractBaseTest extends TestCase { } public void testMkdirsFailsForSubdirectoryOfExistingFile() throws Exception { - Path testDir = path("/test/hadoop"); + Path testDir = path("testMkdirsFailsForSubdirectoryOfExistingFile"); assertFalse(fs.exists(testDir)); assertTrue(fs.mkdirs(testDir)); assertTrue(fs.exists(testDir)); - createFile(path("/test/hadoop/file")); + createFile(path("testMkdirsFailsForSubdirectoryOfExistingFile/file")); - Path testSubDir = path("/test/hadoop/file/subdir"); + Path testSubDir = path( + "testMkdirsFailsForSubdirectoryOfExistingFile/file/subdir"); try { fs.mkdirs(testSubDir); fail("Should throw IOException."); @@ -167,7 +211,8 @@ public abstract class FileSystemContractBaseTest extends TestCase { // file missing execute permission. } - Path testDeepSubDir = path("/test/hadoop/file/deep/sub/dir"); + Path testDeepSubDir = path( + "testMkdirsFailsForSubdirectoryOfExistingFile/file/deep/sub/dir"); try { fs.mkdirs(testDeepSubDir); fail("Should throw IOException."); @@ -190,7 +235,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { String oldUmask = conf.get(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY); try { conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, TEST_UMASK); - final Path dir = path("/test/newDir"); + final Path dir = path("newDir"); assertTrue(fs.mkdirs(dir, new FsPermission((short) 0777))); FileStatus status = fs.getFileStatus(dir); assertTrue(status.isDirectory()); @@ -223,7 +268,8 @@ public abstract class FileSystemContractBaseTest extends TestCase { public void testGetFileStatusThrowsExceptionForNonExistentFile() throws Exception { try { - fs.getFileStatus(path("/test/hadoop/file")); + fs.getFileStatus( + path("testGetFileStatusThrowsExceptionForNonExistentFile/file")); fail("Should throw FileNotFoundException"); } catch (FileNotFoundException e) { // expected @@ -232,7 +278,8 @@ public abstract class FileSystemContractBaseTest extends TestCase { public void testListStatusThrowsExceptionForNonExistentFile() throws Exception { try { - fs.listStatus(path("/test/hadoop/file")); + fs.listStatus( + path("testListStatusThrowsExceptionForNonExistentFile/file")); fail("Should throw FileNotFoundException"); } catch (FileNotFoundException fnfe) { // expected @@ -240,30 +287,32 @@ public abstract class FileSystemContractBaseTest extends TestCase { } public void testListStatus() throws Exception { - Path[] testDirs = { path("/test/hadoop/a"), - path("/test/hadoop/b"), - path("/test/hadoop/c/1"), }; + final Path[] testDirs = { + path("testListStatus/a"), + path("testListStatus/b"), + path("testListStatus/c/1") + }; assertFalse(fs.exists(testDirs[0])); for (Path path : testDirs) { assertTrue(fs.mkdirs(path)); } - FileStatus[] paths = fs.listStatus(path("/test")); + FileStatus[] paths = fs.listStatus(path(".")); assertEquals(1, paths.length); - assertEquals(path("/test/hadoop"), paths[0].getPath()); + assertEquals(path("testListStatus"), paths[0].getPath()); - paths = fs.listStatus(path("/test/hadoop")); + paths = fs.listStatus(path("testListStatus")); assertEquals(3, paths.length); ArrayList list = new ArrayList(); for (FileStatus fileState : paths) { list.add(fileState.getPath()); } - assertTrue(list.contains(path("/test/hadoop/a"))); - assertTrue(list.contains(path("/test/hadoop/b"))); - assertTrue(list.contains(path("/test/hadoop/c"))); + assertTrue(list.contains(path("testListStatus/a"))); + assertTrue(list.contains(path("testListStatus/b"))); + assertTrue(list.contains(path("testListStatus/c"))); - paths = fs.listStatus(path("/test/hadoop/a")); + paths = fs.listStatus(path("testListStatus/a")); assertEquals(0, paths.length); } @@ -294,12 +343,12 @@ public abstract class FileSystemContractBaseTest extends TestCase { * @throws IOException on IO failures */ protected void writeReadAndDelete(int len) throws IOException { - Path path = path("/test/hadoop/file"); + Path path = path("writeReadAndDelete/file"); writeAndRead(path, data, len, false, true); } public void testOverwrite() throws IOException { - Path path = path("/test/hadoop/file"); + Path path = path("testOverwrite/file"); fs.mkdirs(path.getParent()); @@ -325,7 +374,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { } public void testWriteInNonExistentDirectory() throws IOException { - Path path = path("/test/hadoop/file"); + Path path = path("testWriteInNonExistentDirectory/file"); assertFalse("Parent exists", fs.exists(path.getParent())); createFile(path); @@ -335,15 +384,15 @@ public abstract class FileSystemContractBaseTest extends TestCase { } public void testDeleteNonExistentFile() throws IOException { - Path path = path("/test/hadoop/file"); + Path path = path("testDeleteNonExistentFile/file"); assertFalse("Path exists: " + path, fs.exists(path)); assertFalse("No deletion", fs.delete(path, true)); } public void testDeleteRecursively() throws IOException { - Path dir = path("/test/hadoop"); - Path file = path("/test/hadoop/file"); - Path subdir = path("/test/hadoop/subdir"); + Path dir = path("testDeleteRecursively"); + Path file = path("testDeleteRecursively/file"); + Path subdir = path("testDeleteRecursively/subdir"); createFile(file); assertTrue("Created subdir", fs.mkdirs(subdir)); @@ -369,7 +418,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { } public void testDeleteEmptyDirectory() throws IOException { - Path dir = path("/test/hadoop"); + Path dir = path("testDeleteEmptyDirectory"); assertTrue(fs.mkdirs(dir)); assertTrue("Dir exists", fs.exists(dir)); assertTrue("Deleted", fs.delete(dir, false)); @@ -379,26 +428,26 @@ public abstract class FileSystemContractBaseTest extends TestCase { public void testRenameNonExistentPath() throws Exception { if (!renameSupported()) return; - Path src = path("/test/hadoop/path"); - Path dst = path("/test/new/newpath"); + Path src = path("testRenameNonExistentPath/path"); + Path dst = path("testRenameNonExistentPathNew/newpath"); rename(src, dst, false, false, false); } public void testRenameFileMoveToNonExistentDirectory() throws Exception { if (!renameSupported()) return; - Path src = path("/test/hadoop/file"); + Path src = path("testRenameFileMoveToNonExistentDirectory/file"); createFile(src); - Path dst = path("/test/new/newfile"); + Path dst = path("testRenameFileMoveToNonExistentDirectoryNew/newfile"); rename(src, dst, false, true, false); } public void testRenameFileMoveToExistingDirectory() throws Exception { if (!renameSupported()) return; - Path src = path("/test/hadoop/file"); + Path src = path("testRenameFileMoveToExistingDirectory/file"); createFile(src); - Path dst = path("/test/new/newfile"); + Path dst = path("testRenameFileMoveToExistingDirectoryNew/newfile"); fs.mkdirs(dst.getParent()); rename(src, dst, true, false, true); } @@ -406,9 +455,9 @@ public abstract class FileSystemContractBaseTest extends TestCase { public void testRenameFileAsExistingFile() throws Exception { if (!renameSupported()) return; - Path src = path("/test/hadoop/file"); + Path src = path("testRenameFileAsExistingFile/file"); createFile(src); - Path dst = path("/test/new/newfile"); + Path dst = path("testRenameFileAsExistingFileNew/newfile"); createFile(dst); rename(src, dst, false, true, true); } @@ -416,83 +465,81 @@ public abstract class FileSystemContractBaseTest extends TestCase { public void testRenameFileAsExistingDirectory() throws Exception { if (!renameSupported()) return; - Path src = path("/test/hadoop/file"); + Path src = path("testRenameFileAsExistingDirectory/file"); createFile(src); - Path dst = path("/test/new/newdir"); + Path dst = path("testRenameFileAsExistingDirectoryNew/newdir"); fs.mkdirs(dst); rename(src, dst, true, false, true); - assertIsFile(path("/test/new/newdir/file")); + assertIsFile(path("testRenameFileAsExistingDirectoryNew/newdir/file")); } public void testRenameDirectoryMoveToNonExistentDirectory() throws Exception { if (!renameSupported()) return; - Path src = path("/test/hadoop/dir"); + Path src = path("testRenameDirectoryMoveToNonExistentDirectory/dir"); fs.mkdirs(src); - Path dst = path("/test/new/newdir"); + Path dst = path("testRenameDirectoryMoveToNonExistentDirectoryNew/newdir"); rename(src, dst, false, true, false); } public void testRenameDirectoryMoveToExistingDirectory() throws Exception { if (!renameSupported()) return; - - Path src = path("/test/hadoop/dir"); + Path src = path("testRenameDirectoryMoveToExistingDirectory/dir"); fs.mkdirs(src); - createFile(path("/test/hadoop/dir/file1")); - createFile(path("/test/hadoop/dir/subdir/file2")); + createFile(path(src + "/file1")); + createFile(path(src + "/subdir/file2")); - Path dst = path("/test/new/newdir"); + Path dst = path("testRenameDirectoryMoveToExistingDirectoryNew/newdir"); fs.mkdirs(dst.getParent()); rename(src, dst, true, false, true); assertFalse("Nested file1 exists", - fs.exists(path("/test/hadoop/dir/file1"))); + fs.exists(path(src + "/file1"))); assertFalse("Nested file2 exists", - fs.exists(path("/test/hadoop/dir/subdir/file2"))); + fs.exists(path(src + "/subdir/file2"))); assertTrue("Renamed nested file1 exists", - fs.exists(path("/test/new/newdir/file1"))); + fs.exists(path(dst + "/file1"))); assertTrue("Renamed nested exists", - fs.exists(path("/test/new/newdir/subdir/file2"))); + fs.exists(path(dst + "/subdir/file2"))); } public void testRenameDirectoryAsExistingFile() throws Exception { if (!renameSupported()) return; - Path src = path("/test/hadoop/dir"); + Path src = path("testRenameDirectoryAsExistingFile/dir"); fs.mkdirs(src); - Path dst = path("/test/new/newfile"); + Path dst = path("testRenameDirectoryAsExistingFileNew/newfile"); createFile(dst); rename(src, dst, false, true, true); } public void testRenameDirectoryAsExistingDirectory() throws Exception { if (!renameSupported()) return; - - Path src = path("/test/hadoop/dir"); + final Path src = path("testRenameDirectoryAsExistingDirectory/dir"); fs.mkdirs(src); - createFile(path("/test/hadoop/dir/file1")); - createFile(path("/test/hadoop/dir/subdir/file2")); - - Path dst = path("/test/new/newdir"); + createFile(path(src + "/file1")); + createFile(path(src + "/subdir/file2")); + + final Path dst = path("testRenameDirectoryAsExistingDirectoryNew/newdir"); fs.mkdirs(dst); rename(src, dst, true, false, true); assertTrue("Destination changed", - fs.exists(path("/test/new/newdir/dir"))); + fs.exists(path(dst + "/dir"))); assertFalse("Nested file1 exists", - fs.exists(path("/test/hadoop/dir/file1"))); + fs.exists(path(src + "/file1"))); assertFalse("Nested file2 exists", - fs.exists(path("/test/hadoop/dir/subdir/file2"))); + fs.exists(path(src + "/dir/subdir/file2"))); assertTrue("Renamed nested file1 exists", - fs.exists(path("/test/new/newdir/dir/file1"))); + fs.exists(path(dst + "/dir/file1"))); assertTrue("Renamed nested exists", - fs.exists(path("/test/new/newdir/dir/subdir/file2"))); + fs.exists(path(dst + "/dir/subdir/file2"))); } public void testInputStreamClosedTwice() throws IOException { //HADOOP-4760 according to Closeable#close() closing already-closed //streams should have no effect. - Path src = path("/test/hadoop/file"); + Path src = path("testInputStreamClosedTwice/file"); createFile(src); FSDataInputStream in = fs.open(src); in.close(); @@ -502,18 +549,13 @@ public abstract class FileSystemContractBaseTest extends TestCase { public void testOutputStreamClosedTwice() throws IOException { //HADOOP-4760 according to Closeable#close() closing already-closed //streams should have no effect. - Path src = path("/test/hadoop/file"); + Path src = path("testOutputStreamClosedTwice/file"); FSDataOutputStream out = fs.create(src); out.writeChar('H'); //write some data out.close(); out.close(); } - - protected Path path(String pathString) { - return new Path(pathString).makeQualified(fs.getUri(), - fs.getWorkingDirectory()); - } - + protected void createFile(Path path) throws IOException { FSDataOutputStream out = fs.create(path); out.write(data, 0, data.length); @@ -541,7 +583,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { byte[] filedata1 = dataset(blockSize * 2, 'A', 26); byte[] filedata2 = dataset(blockSize * 2, 'a', 26); - Path path = path("/test/hadoop/file-overwrite"); + Path path = path("testOverWriteAndRead/file-overwrite"); writeAndRead(path, filedata1, blockSize, true, false); writeAndRead(path, filedata2, blockSize, true, false); writeAndRead(path, filedata1, blockSize * 2, true, false); @@ -561,7 +603,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { LOG.info("Skipping test"); return; } - String mixedCaseFilename = "/test/UPPER.TXT"; + String mixedCaseFilename = "testFilesystemIsCaseSensitive"; Path upper = path(mixedCaseFilename); Path lower = path(StringUtils.toLowerCase(mixedCaseFilename)); assertFalse("File exists" + upper, fs.exists(upper)); @@ -592,7 +634,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { * @throws Exception on failures */ public void testZeroByteFilesAreFiles() throws Exception { - Path src = path("/test/testZeroByteFilesAreFiles"); + Path src = path("testZeroByteFilesAreFiles"); //create a zero byte file FSDataOutputStream out = fs.create(src); out.close(); @@ -605,7 +647,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { * @throws Exception on failures */ public void testMultiByteFilesAreFiles() throws Exception { - Path src = path("/test/testMultiByteFilesAreFiles"); + Path src = path("testMultiByteFilesAreFiles"); FSDataOutputStream out = fs.create(src); out.writeUTF("testMultiByteFilesAreFiles"); out.close(); @@ -629,10 +671,14 @@ public abstract class FileSystemContractBaseTest extends TestCase { * @throws Exception on failures */ public void testRenameRootDirForbidden() throws Exception { + if (!rootDirTestEnabled()) { + return; + } + if (!renameSupported()) return; rename(path("/"), - path("/test/newRootDir"), + path("testRenameRootDirForbidden"), false, true, false); } @@ -644,7 +690,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { public void testRenameChildDirForbidden() throws Exception { if (!renameSupported()) return; LOG.info("testRenameChildDirForbidden"); - Path parentdir = path("/test/parentdir"); + Path parentdir = path("testRenameChildDirForbidden"); fs.mkdirs(parentdir); Path childFile = new Path(parentdir, "childfile"); createFile(childFile); @@ -663,9 +709,9 @@ public abstract class FileSystemContractBaseTest extends TestCase { */ public void testRenameToDirWithSamePrefixAllowed() throws Throwable { if (!renameSupported()) return; - Path parentdir = path("test/parentdir"); + final Path parentdir = path("testRenameToDirWithSamePrefixAllowed"); fs.mkdirs(parentdir); - Path dest = path("test/parentdirdest"); + final Path dest = path("testRenameToDirWithSamePrefixAllowedDest"); rename(parentdir, dest, true, false, true); } @@ -677,7 +723,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { if (!renameSupported()) { return; } - Path parentdir = path("test/parentdir"); + Path parentdir = path("testRenameDirToSelf"); fs.mkdirs(parentdir); Path child = new Path(parentdir, "child"); createFile(child); @@ -696,7 +742,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { if (!renameSupported()) { return; } - Path testdir = path("test/dir"); + Path testdir = path("testMoveDirUnderParent"); fs.mkdirs(testdir); Path parent = testdir.getParent(); //the outcome here is ambiguous, so is not checked @@ -711,7 +757,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { */ public void testRenameFileToSelf() throws Throwable { if (!renameSupported()) return; - Path filepath = path("test/file"); + Path filepath = path("testRenameFileToSelf"); createFile(filepath); //HDFS expects rename src, src -> true rename(filepath, filepath, true, true, true); @@ -725,7 +771,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { */ public void testMoveFileUnderParent() throws Throwable { if (!renameSupported()) return; - Path filepath = path("test/file"); + Path filepath = path("testMoveFileUnderParent"); createFile(filepath); //HDFS expects rename src, src -> true rename(filepath, filepath, true, true, true); @@ -734,15 +780,23 @@ public abstract class FileSystemContractBaseTest extends TestCase { } public void testLSRootDir() throws Throwable { + if (!rootDirTestEnabled()) { + return; + } + Path dir = path("/"); - Path child = path("/test"); + Path child = path("/FileSystemContractBaseTest"); createFile(child); assertListFilesFinds(dir, child); } public void testListStatusRootDir() throws Throwable { + if (!rootDirTestEnabled()) { + return; + } + Path dir = path("/"); - Path child = path("/test"); + Path child = path("/FileSystemContractBaseTest"); createFile(child); assertListStatusFinds(dir, child); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestRawLocalFileSystemContract.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestRawLocalFileSystemContract.java index 036fb6af978..b023c091d3a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestRawLocalFileSystemContract.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestRawLocalFileSystemContract.java @@ -33,6 +33,8 @@ public class TestRawLocalFileSystemContract extends FileSystemContractBaseTest { private static final Logger LOG = LoggerFactory.getLogger(TestRawLocalFileSystemContract.class); + private final static Path TEST_BASE_DIR = + new Path(GenericTestUtils.getTempPath("")); @Before public void setUp() throws Exception { @@ -51,21 +53,25 @@ public class TestRawLocalFileSystemContract extends FileSystemContractBaseTest { return false; } + /** + * Disabling testing root operation. + * + * Writing to root directory on the local file system may get permission + * denied exception, or even worse, delete/overwrite files accidentally. + */ + @Override + protected boolean rootDirTestEnabled() { + return false; + } + @Override public String getDefaultWorkingDirectory() { return fs.getWorkingDirectory().toUri().getPath(); } @Override - protected Path path(String pathString) { - // For testWorkingDirectory - if (pathString.equals(getDefaultWorkingDirectory()) || - pathString.equals(".") || pathString.equals("..")) { - return super.path(pathString); - } - - return new Path(GenericTestUtils.getTempPath(pathString)). - makeQualified(fs.getUri(), fs.getWorkingDirectory()); + protected Path getTestBaseDir() { + return TEST_BASE_DIR; } @Override diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java index f39ec81a79a..6fcf4c7d2e7 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java @@ -57,30 +57,13 @@ public class ITestS3AFileSystemContract extends FileSystemContractBaseTest { fs = S3ATestUtils.createTestFileSystem(conf); basePath = fs.makeQualified( - S3ATestUtils.createTestPath(new Path("/s3afilesystemcontract"))); + S3ATestUtils.createTestPath(new Path("s3afilesystemcontract"))); super.setUp(); } - /** - * This path explicitly places all absolute paths under the per-test suite - * path directory; this allows the test to run in parallel. - * @param pathString path string as input - * @return a qualified path string. - */ - protected Path path(String pathString) { - if (pathString.startsWith("/")) { - return fs.makeQualified(new Path(basePath, pathString)); - } else { - return super.path(pathString); - } - } - @Override - protected void tearDown() throws Exception { - if (fs != null) { - fs.delete(basePath, true); - } - super.tearDown(); + public Path getTestBaseDir() { + return basePath; } @Override @@ -94,22 +77,22 @@ public class ITestS3AFileSystemContract extends FileSystemContractBaseTest { return; } - Path src = path("/test/hadoop/dir"); + Path src = path("testRenameDirectoryAsExisting/dir"); fs.mkdirs(src); - createFile(path("/test/hadoop/dir/file1")); - createFile(path("/test/hadoop/dir/subdir/file2")); + createFile(path(src + "/file1")); + createFile(path(src + "/subdir/file2")); - Path dst = path("/test/new/newdir"); + Path dst = path("testRenameDirectoryAsExistingNew/newdir"); fs.mkdirs(dst); rename(src, dst, true, false, true); assertFalse("Nested file1 exists", - fs.exists(path("/test/hadoop/dir/file1"))); + fs.exists(path(src + "/file1"))); assertFalse("Nested file2 exists", - fs.exists(path("/test/hadoop/dir/subdir/file2"))); + fs.exists(path(src + "/subdir/file2"))); assertTrue("Renamed nested file1 exists", - fs.exists(path("/test/new/newdir/file1"))); + fs.exists(path(dst + "/file1"))); assertTrue("Renamed nested exists", - fs.exists(path("/test/new/newdir/subdir/file2"))); + fs.exists(path(dst + "/subdir/file2"))); } // @Override From 4c66a8d19b7d503095ad27aeed39d62238b9cb47 Mon Sep 17 00:00:00 2001 From: Ray Chiang Date: Tue, 14 Mar 2017 14:37:18 -0700 Subject: [PATCH 072/188] YARN-6331. Fix flakiness in TestFairScheduler#testDumpState. (Yufei Gu via rchiang) --- .../resourcemanager/scheduler/fair/TestFairScheduler.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 537d3d0d392..baf74346fed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -5201,6 +5201,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { FSParentQueue parent = scheduler.getQueueManager().getParentQueue("parent", false); parent.setMaxShare(resource); + parent.updateDemand(); String parentQueueString = "{Name: root.parent," + " Weight: ," @@ -5210,7 +5211,7 @@ public class TestFairScheduler extends FairSchedulerTestBase { + " MaxShare: ," + " MinShare: ," + " ResourceUsage: ," - + " Demand: ," + + " Demand: ," + " MaxAMShare: 0.5," + " Runnable: 0}"; From cc1292e73acd39c1f1023ad4841ffe30176f7daf Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Tue, 14 Mar 2017 16:41:10 -0700 Subject: [PATCH 073/188] HDFS-9705. Refine the behaviour of getFileChecksum when length = 0. Contributed by Kai Zheng and SammiChen. --- .../org/apache/hadoop/hdfs/DFSClient.java | 10 +++-- .../hadoop/hdfs/FileChecksumHelper.java | 42 +++++++++++-------- .../server/datanode/BlockChecksumHelper.java | 6 +-- .../hdfs/TestDistributedFileSystem.java | 22 ++++++---- 4 files changed, 48 insertions(+), 32 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index ae1d8217b9c..aaf8bdd6ba4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -1731,10 +1731,14 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, checkOpen(); Preconditions.checkArgument(length >= 0); - LocatedBlocks blockLocations = getBlockLocations(src, length); + LocatedBlocks blockLocations = null; + FileChecksumHelper.FileChecksumComputer maker = null; + ErasureCodingPolicy ecPolicy = null; + if (length > 0) { + blockLocations = getBlockLocations(src, length); + ecPolicy = blockLocations.getErasureCodingPolicy(); + } - FileChecksumHelper.FileChecksumComputer maker; - ErasureCodingPolicy ecPolicy = blockLocations.getErasureCodingPolicy(); maker = ecPolicy != null ? new FileChecksumHelper.StripedFileNonStripedChecksumComputer(src, length, blockLocations, namenode, this, ecPolicy) : diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/FileChecksumHelper.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/FileChecksumHelper.java index fe462f27553..689d46d8223 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/FileChecksumHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/FileChecksumHelper.java @@ -95,11 +95,13 @@ final class FileChecksumHelper { this.client = client; this.remaining = length; - if (src.contains(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR_SEPARATOR)) { - this.remaining = Math.min(length, blockLocations.getFileLength()); - } - this.locatedBlocks = blockLocations.getLocatedBlocks(); + if (blockLocations != null) { + if (src.contains(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR_SEPARATOR)) { + this.remaining = Math.min(length, blockLocations.getFileLength()); + } + this.locatedBlocks = blockLocations.getLocatedBlocks(); + } } String getSrc() { @@ -203,9 +205,23 @@ final class FileChecksumHelper { * @throws IOException */ void compute() throws IOException { - checksumBlocks(); - - fileChecksum = makeFinalResult(); + /** + * request length is 0 or the file is empty, return one with the + * magic entry that matches what previous hdfs versions return. + */ + if (locatedBlocks == null || locatedBlocks.isEmpty()) { + // Explicitly specified here in case the default DataOutputBuffer + // buffer length value is changed in future. This matters because the + // fixed value 32 has to be used to repeat the magic value for previous + // HDFS version. + final int lenOfZeroBytes = 32; + byte[] emptyBlockMd5 = new byte[lenOfZeroBytes]; + MD5Hash fileMD5 = MD5Hash.digest(emptyBlockMd5); + fileChecksum = new MD5MD5CRC32GzipFileChecksum(0, 0, fileMD5); + } else { + checksumBlocks(); + fileChecksum = makeFinalResult(); + } } /** @@ -228,15 +244,7 @@ final class FileChecksumHelper { return new MD5MD5CRC32CastagnoliFileChecksum(bytesPerCRC, crcPerBlock, fileMD5); default: - // If there is no block allocated for the file, - // return one with the magic entry that matches what previous - // hdfs versions return. - if (locatedBlocks.isEmpty()) { - return new MD5MD5CRC32GzipFileChecksum(0, 0, fileMD5); - } - - // we should never get here since the validity was checked - // when getCrcType() was called above. + // we will get here when crcType is "NULL". return null; } } @@ -412,7 +420,7 @@ final class FileChecksumHelper { } /** - * Striped file checksum computing. + * Non-striped checksum computing for striped files. */ static class StripedFileNonStripedChecksumComputer extends FileChecksumComputer { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockChecksumHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockChecksumHelper.java index f54978546ec..e99911bdef7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockChecksumHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockChecksumHelper.java @@ -66,7 +66,7 @@ final class BlockChecksumHelper { } /** - * The abstract base block checksum computer. + * The abstract block checksum computer. */ static abstract class AbstractBlockChecksumComputer { private final DataNode datanode; @@ -139,7 +139,7 @@ final class BlockChecksumHelper { } /** - * The abstract base block checksum computer. + * The abstract base block checksum computer, mainly for replicated blocks. */ static abstract class BlockChecksumComputer extends AbstractBlockChecksumComputer { @@ -534,4 +534,4 @@ final class BlockChecksumHelper { } } } -} \ No newline at end of file +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java index 69248d98fe6..76545316b6b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java @@ -1030,13 +1030,6 @@ public class TestDistributedFileSystem { out.close(); } - // verify the magic val for zero byte files - { - final FileChecksum zeroChecksum = hdfs.getFileChecksum(zeroByteFile); - assertEquals(zeroChecksum.toString(), - "MD5-of-0MD5-of-0CRC32:70bc8f4b72a86921468bf8e8441dce51"); - } - //write another file final Path bar = new Path(dir, "bar" + n); { @@ -1045,8 +1038,19 @@ public class TestDistributedFileSystem { out.write(data); out.close(); } - - { //verify checksum + + { + final FileChecksum zeroChecksum = hdfs.getFileChecksum(zeroByteFile); + final String magicValue = + "MD5-of-0MD5-of-0CRC32:70bc8f4b72a86921468bf8e8441dce51"; + // verify the magic val for zero byte files + assertEquals(magicValue, zeroChecksum.toString()); + + //verify checksums for empty file and 0 request length + final FileChecksum checksumWith0 = hdfs.getFileChecksum(bar, 0); + assertEquals(zeroChecksum, checksumWith0); + + //verify checksum final FileChecksum barcs = hdfs.getFileChecksum(bar); final int barhashcode = barcs.hashCode(); assertEquals(hdfsfoocs.hashCode(), barhashcode); From 2d0e24eb01cec6f4554671b54729717510711c63 Mon Sep 17 00:00:00 2001 From: Sunil G Date: Wed, 15 Mar 2017 14:48:05 +0530 Subject: [PATCH 074/188] YARN-6328. Fix a spelling mistake in CapacityScheduler. Contributed by Jin Yibo. This closes #202 --- .../resourcemanager/scheduler/capacity/CapacityScheduler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index f6e79426865..d3186da5b4c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -2114,7 +2114,7 @@ public class CapacityScheduler extends // Get the default priority for the Queue. If Queue is non-existent, // then - // use default priority. Do it only if user doesnt have any default. + // use default priority. Do it only if user doesn't have any default. if (null == appPriority) { appPriority = this.queueManager.getDefaultPriorityForQueue(queueName); } From 4416a07c9c57af01ffbd2e43afcde2a1ba94c736 Mon Sep 17 00:00:00 2001 From: Yiqun Lin Date: Wed, 15 Mar 2017 18:05:03 +0800 Subject: [PATCH 075/188] HDFS-11420. Edit file should not be processed by the same type processor in OfflineEditsViewer. Contributed by Yiqun Lin. --- .../OfflineEditsViewer.java | 19 +++++++++++++++++-- .../src/site/markdown/HdfsEditsViewer.md | 2 ++ .../TestOfflineEditsViewer.java | 18 ++++++++++++++++++ 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsViewer.java index 107881ff590..f075ed29ec4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsViewer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsViewer.java @@ -22,6 +22,7 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsLoader.OfflineEditsLoaderFactory; +import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; @@ -55,7 +56,9 @@ public class OfflineEditsViewer extends Configured implements Tool { "Required command line arguments:\n" + "-i,--inputFile edits file to process, xml (case\n" + " insensitive) extension means XML format,\n" + - " any other filename means binary format\n" + + " any other filename means binary format.\n" + + " XML/Binary format input file is not allowed\n" + + " to be processed by the same type processor.\n" + "-o,--outputFile Name of output file. If the specified\n" + " file exists, it will be overwritten,\n" + " format of the file is determined\n" + @@ -132,12 +135,24 @@ public class OfflineEditsViewer extends Configured implements Tool { System.out.println("input [" + inputFileName + "]"); System.out.println("output [" + outputFileName + "]"); } + + boolean xmlInput = StringUtils.toLowerCase(inputFileName).endsWith(".xml"); + if (xmlInput && StringUtils.equalsIgnoreCase("xml", processor)) { + System.err.println("XML format input file is not allowed" + + " to be processed by XML processor."); + return -1; + } else if(!xmlInput && StringUtils.equalsIgnoreCase("binary", processor)) { + System.err.println("Binary format input file is not allowed" + + " to be processed by Binary processor."); + return -1; + } + try { if (visitor == null) { visitor = OfflineEditsVisitorFactory.getEditsVisitor( outputFileName, processor, flags.getPrintToScreen()); } - boolean xmlInput = inputFileName.toLowerCase().endsWith(".xml"); + OfflineEditsLoader loader = OfflineEditsLoaderFactory. createLoader(visitor, inputFileName, xmlInput, flags); loader.loadEdits(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsEditsViewer.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsEditsViewer.md index 5e069bbacea..4ab07ce2143 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsEditsViewer.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsEditsViewer.md @@ -30,6 +30,8 @@ Input formats supported: 2. **xml**: XML format, as produced by xml processor, used if filename has `.xml` (case insensitive) extension +Note: XML/Binary format input file is not allowed to be processed by the same type processor. + The Offline Edits Viewer provides several output processors (unless stated otherwise the output of the processor can be converted back to original edits file): 1. **binary**: native binary format that Hadoop uses internally diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java index a21bc8f1299..bbad73c0418 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java @@ -338,4 +338,22 @@ public class TestOfflineEditsViewer { } } } + + @Test + public void testProcessorWithSameTypeFormatFile() throws IOException { + String edits = nnHelper.generateEdits(); + LOG.info("Generated edits=" + edits); + String binaryEdits = folder.newFile("binaryEdits").getAbsolutePath(); + String editsParsedXml = folder.newFile("editsParsed.xml").getAbsolutePath(); + String editsReparsedXml = folder.newFile("editsReparsed.xml") + .getAbsolutePath(); + + // Binary format input file is not allowed to be processed + // by Binary processor. + assertEquals(-1, runOev(edits, binaryEdits, "binary", false)); + // parse to XML then back to XML + assertEquals(0, runOev(edits, editsParsedXml, "xml", false)); + // XML format input file is not allowed to be processed by XML processor. + assertEquals(-1, runOev(editsParsedXml, editsReparsedXml, "xml", false)); + } } From bb6a214cd34b9c87b0ba68205c680024ad294587 Mon Sep 17 00:00:00 2001 From: Rohith Sharma K S Date: Wed, 15 Mar 2017 15:53:57 +0530 Subject: [PATCH 076/188] YARN-6336. Jenkins report YARN new UI build failure. Contributed by Sunil G. --- .../hadoop-yarn/hadoop-yarn-ui/pom.xml | 13 ++++++------- .../hadoop-yarn-ui/src/main/webapp/package.json | 6 ++++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml index ae7ab0d3c71..31d2f5e6880 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml @@ -31,7 +31,7 @@ pom ${basedir}/target/src/main/webapp - node + ${basedir}/target/src/main/webapp/node/node v4.4.5 2.15.5 false @@ -169,12 +169,11 @@ ${webappTgtDir} - ember + ${node.executable} - build - -prod - --output-path - ${basedir}/target/dist + node/npm/bin/npm-cli + run + build:mvn @@ -187,7 +186,7 @@ maven-war-plugin ${basedir}/src/main/webapp/WEB-INF/web.xml - ${basedir}/target/dist + ${webappTgtDir}/dist diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package.json index 573e5051283..105dad3fb03 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package.json +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package.json @@ -8,8 +8,10 @@ "test": "tests" }, "scripts": { - "build": "ember build", - "start": "ember server" + "build": "TMPDIR=tmp node ./node_modules/ember-cli/bin/ember build", + "start": "TMPDIR=tmp node ./node_modules/ember-cli/bin/ember server", + + "build:mvn": "TMPDIR=tmp node/node ./node_modules/ember-cli/bin/ember build -prod" }, "repository": "", "engines": { From 221783b35bb78bda909c2417c7b9687d491fa196 Mon Sep 17 00:00:00 2001 From: Junping Du Date: Wed, 15 Mar 2017 11:48:09 -0700 Subject: [PATCH 077/188] YARN-6294. ATS client should better handle Socket closed case. Contributed by Li Lu. --- .../apache/hadoop/yarn/client/api/impl/TimelineConnector.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java index b5b5f7759ee..bb29d6cbb4b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.lang.reflect.UndeclaredThrowableException; import java.net.ConnectException; import java.net.HttpURLConnection; +import java.net.SocketException; import java.net.SocketTimeoutException; import java.net.URI; import java.net.URL; @@ -390,7 +391,8 @@ public class TimelineConnector extends AbstractService { // Only retry on connection exceptions return (e instanceof ClientHandlerException) && (e.getCause() instanceof ConnectException - || e.getCause() instanceof SocketTimeoutException); + || e.getCause() instanceof SocketTimeoutException + || e.getCause() instanceof SocketException); } }; try { From 615ac09499dc0b391cbb99bb0e9877959a9173a6 Mon Sep 17 00:00:00 2001 From: Chen Liang Date: Wed, 15 Mar 2017 12:28:22 -0700 Subject: [PATCH 078/188] HDFS-11419. DFSTopologyNodeImpl#chooseRandom optimizations. Contributed by Chen Liang. --- .../hadoop/hdfs/net/DFSTopologyNodeImpl.java | 121 +++++++++++------- .../hdfs/net/TestDFSNetworkTopology.java | 64 ++++++++- 2 files changed, 137 insertions(+), 48 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSTopologyNodeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSTopologyNodeImpl.java index 92bb30395fc..6d80db5af89 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSTopologyNodeImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSTopologyNodeImpl.java @@ -27,7 +27,6 @@ import org.apache.hadoop.net.Node; import java.util.EnumMap; import java.util.EnumSet; import java.util.HashMap; -import java.util.Map; /** * The HDFS-specific representation of a network topology inner node. The @@ -76,32 +75,58 @@ public class DFSTopologyNodeImpl extends InnerNodeImpl { private final HashMap > childrenStorageInfo; + /** + * This map stores storage type counts of the subtree. We can always get this + * info by iterate over the childrenStorageInfo variable. But for optimization + * purpose, we store this info directly to avoid the iteration. + */ + private final EnumMap storageTypeCounts; + DFSTopologyNodeImpl(String path) { super(path); childrenStorageInfo = new HashMap<>(); + storageTypeCounts = new EnumMap<>(StorageType.class); } DFSTopologyNodeImpl( String name, String location, InnerNode parent, int level) { super(name, location, parent, level); childrenStorageInfo = new HashMap<>(); + storageTypeCounts = new EnumMap<>(StorageType.class); } public int getSubtreeStorageCount(StorageType type) { - int res = 0; - for (Map.Entry> entry : - childrenStorageInfo.entrySet()) { - if (entry.getValue().containsKey(type)) { - res += entry.getValue().get(type); - } + if (storageTypeCounts.containsKey(type)) { + return storageTypeCounts.get(type); + } else { + return 0; } - return res; } int getNumOfChildren() { return children.size(); } + private void incStorageTypeCount(StorageType type) { + // no locking because the caller is synchronized already + if (storageTypeCounts.containsKey(type)) { + storageTypeCounts.put(type, storageTypeCounts.get(type)+1); + } else { + storageTypeCounts.put(type, 1); + } + } + + private void decStorageTypeCount(StorageType type) { + // no locking because the caller is synchronized already + int current = storageTypeCounts.get(type); + current -= 1; + if (current == 0) { + storageTypeCounts.remove(type); + } else { + storageTypeCounts.put(type, current); + } + } + @Override public boolean add(Node n) { if (!isAncestor(n)) { @@ -130,14 +155,13 @@ public class DFSTopologyNodeImpl extends InnerNodeImpl { } children.add(n); numOfLeaves++; - synchronized (childrenStorageInfo) { - if (!childrenStorageInfo.containsKey(dnDescriptor.getName())) { - childrenStorageInfo.put( - dnDescriptor.getName(), new EnumMap<>(StorageType.class)); - } - for (StorageType st : dnDescriptor.getStorageTypes()) { - childrenStorageInfo.get(dnDescriptor.getName()).put(st, 1); - } + if (!childrenStorageInfo.containsKey(dnDescriptor.getName())) { + childrenStorageInfo.put( + dnDescriptor.getName(), new EnumMap<>(StorageType.class)); + } + for (StorageType st : dnDescriptor.getStorageTypes()) { + childrenStorageInfo.get(dnDescriptor.getName()).put(st, 1); + incStorageTypeCount(st); } return true; } else { @@ -153,25 +177,26 @@ public class DFSTopologyNodeImpl extends InnerNodeImpl { // add n to the subtree of the next ancestor node if (parentNode.add(n)) { numOfLeaves++; - synchronized (childrenStorageInfo) { - if (!childrenStorageInfo.containsKey(parentNode.getName())) { - childrenStorageInfo.put( - parentNode.getName(), new EnumMap<>(StorageType.class)); - for (StorageType st : dnDescriptor.getStorageTypes()) { - childrenStorageInfo.get(parentNode.getName()).put(st, 1); - } - } else { - EnumMap currentCount = - childrenStorageInfo.get(parentNode.getName()); - for (StorageType st : dnDescriptor.getStorageTypes()) { - if (currentCount.containsKey(st)) { - currentCount.put(st, currentCount.get(st) + 1); - } else { - currentCount.put(st, 1); - } + if (!childrenStorageInfo.containsKey(parentNode.getName())) { + childrenStorageInfo.put( + parentNode.getName(), new EnumMap<>(StorageType.class)); + for (StorageType st : dnDescriptor.getStorageTypes()) { + childrenStorageInfo.get(parentNode.getName()).put(st, 1); + } + } else { + EnumMap currentCount = + childrenStorageInfo.get(parentNode.getName()); + for (StorageType st : dnDescriptor.getStorageTypes()) { + if (currentCount.containsKey(st)) { + currentCount.put(st, currentCount.get(st) + 1); + } else { + currentCount.put(st, 1); } } } + for (StorageType st : dnDescriptor.getStorageTypes()) { + incStorageTypeCount(st); + } return true; } else { return false; @@ -220,8 +245,9 @@ public class DFSTopologyNodeImpl extends InnerNodeImpl { if (children.get(i).getName().equals(n.getName())) { children.remove(i); childrenMap.remove(n.getName()); - synchronized (childrenStorageInfo) { - childrenStorageInfo.remove(dnDescriptor.getName()); + childrenStorageInfo.remove(dnDescriptor.getName()); + for (StorageType st : dnDescriptor.getStorageTypes()) { + decStorageTypeCount(st); } numOfLeaves--; n.setParent(null); @@ -242,20 +268,21 @@ public class DFSTopologyNodeImpl extends InnerNodeImpl { boolean isRemoved = parentNode.remove(n); if (isRemoved) { // if the parent node has no children, remove the parent node too - synchronized (childrenStorageInfo) { - EnumMap currentCount = - childrenStorageInfo.get(parentNode.getName()); - EnumSet toRemove = EnumSet.noneOf(StorageType.class); - for (StorageType st : dnDescriptor.getStorageTypes()) { - int newCount = currentCount.get(st) - 1; - if (newCount == 0) { - toRemove.add(st); - } - currentCount.put(st, newCount); - } - for (StorageType st : toRemove) { - currentCount.remove(st); + EnumMap currentCount = + childrenStorageInfo.get(parentNode.getName()); + EnumSet toRemove = EnumSet.noneOf(StorageType.class); + for (StorageType st : dnDescriptor.getStorageTypes()) { + int newCount = currentCount.get(st) - 1; + if (newCount == 0) { + toRemove.add(st); } + currentCount.put(st, newCount); + } + for (StorageType st : toRemove) { + currentCount.remove(st); + } + for (StorageType st : dnDescriptor.getStorageTypes()) { + decStorageTypeCount(st); } if (parentNode.getNumOfChildren() == 0) { for(int i=0; i < children.size(); i++) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/net/TestDFSNetworkTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/net/TestDFSNetworkTopology.java index 32ecf88670e..30ef2ac7f27 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/net/TestDFSNetworkTopology.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/net/TestDFSNetworkTopology.java @@ -229,7 +229,6 @@ public class TestDFSNetworkTopology { assertEquals(1, (int)l1info.get("d2").get(StorageType.DISK)); assertEquals(2, (int)l1info.get("d3").get(StorageType.SSD)); - for (int i = 0; i<4; i++) { CLUSTER.remove(newDD[i]); } @@ -446,4 +445,67 @@ public class TestDFSNetworkTopology { "/l100/d100/r100", null, null, StorageType.DISK); assertNull(n); } + + /** + * Tests getting subtree storage counts, and see whether it is correct when + * we update subtree. + * @throws Exception + */ + @Test + public void testGetSubtreeStorageCount() throws Exception { + // add and remove a node to rack /l2/d3/r1. So all the inner nodes /l2, + // /l2/d3 and /l2/d3/r1 should be affected. /l2/d3/r3 should still be the + // same, only checked as a reference + Node l2 = CLUSTER.getNode("/l2"); + Node l2d3 = CLUSTER.getNode("/l2/d3"); + Node l2d3r1 = CLUSTER.getNode("/l2/d3/r1"); + Node l2d3r3 = CLUSTER.getNode("/l2/d3/r3"); + + assertTrue(l2 instanceof DFSTopologyNodeImpl); + assertTrue(l2d3 instanceof DFSTopologyNodeImpl); + assertTrue(l2d3r1 instanceof DFSTopologyNodeImpl); + assertTrue(l2d3r3 instanceof DFSTopologyNodeImpl); + + DFSTopologyNodeImpl innerl2 = (DFSTopologyNodeImpl)l2; + DFSTopologyNodeImpl innerl2d3 = (DFSTopologyNodeImpl)l2d3; + DFSTopologyNodeImpl innerl2d3r1 = (DFSTopologyNodeImpl)l2d3r1; + DFSTopologyNodeImpl innerl2d3r3 = (DFSTopologyNodeImpl)l2d3r3; + + assertEquals(4, + innerl2.getSubtreeStorageCount(StorageType.DISK)); + assertEquals(2, + innerl2d3.getSubtreeStorageCount(StorageType.DISK)); + assertEquals(1, + innerl2d3r1.getSubtreeStorageCount(StorageType.DISK)); + assertEquals(1, + innerl2d3r3.getSubtreeStorageCount(StorageType.DISK)); + + DatanodeStorageInfo storageInfo = + DFSTestUtil.createDatanodeStorageInfo("StorageID", + "1.2.3.4", "/l2/d3/r1", "newhost"); + DatanodeDescriptor newNode = storageInfo.getDatanodeDescriptor(); + CLUSTER.add(newNode); + + // after adding a storage to /l2/d3/r1, ancestor inner node should have + // DISK count incremented by 1. + assertEquals(5, + innerl2.getSubtreeStorageCount(StorageType.DISK)); + assertEquals(3, + innerl2d3.getSubtreeStorageCount(StorageType.DISK)); + assertEquals(2, + innerl2d3r1.getSubtreeStorageCount(StorageType.DISK)); + assertEquals(1, + innerl2d3r3.getSubtreeStorageCount(StorageType.DISK)); + + CLUSTER.remove(newNode); + + assertEquals(4, + innerl2.getSubtreeStorageCount(StorageType.DISK)); + assertEquals(2, + innerl2d3.getSubtreeStorageCount(StorageType.DISK)); + assertEquals(1, + innerl2d3r1.getSubtreeStorageCount(StorageType.DISK)); + assertEquals(1, + innerl2d3r3.getSubtreeStorageCount(StorageType.DISK)); + } } From d69a82c89c1e857a2a86ff614ae11fb5df993614 Mon Sep 17 00:00:00 2001 From: Hanisha Koneru Date: Wed, 15 Mar 2017 18:01:45 -0700 Subject: [PATCH 079/188] HDFS-11511. Support Timeout when checking single disk. Contributed by Hanisha Koneru. --- LICENSE.txt | 22 +- .../dev-support/findbugsExcludeFile.xml | 12 + .../datanode/checker/AbstractFuture.java | 1295 +++++++++++++++++ .../checker/DatasetVolumeChecker.java | 15 +- .../checker/StorageLocationChecker.java | 1 + .../checker/ThrottledAsyncChecker.java | 37 +- .../datanode/checker/TimeoutFuture.java | 162 +++ .../TestDatasetVolumeCheckerTimeout.java | 134 ++ .../checker/TestThrottledAsyncChecker.java | 14 +- .../TestThrottledAsyncCheckerTimeout.java | 223 +++ 10 files changed, 1905 insertions(+), 10 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/AbstractFuture.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/TimeoutFuture.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestDatasetVolumeCheckerTimeout.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestThrottledAsyncCheckerTimeout.java diff --git a/LICENSE.txt b/LICENSE.txt index d334204aaa8..52da57acb6d 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -2659,4 +2659,24 @@ available under the Creative Commons By Attribution 3.0 License. available upon request from time to time. For the avoidance of doubt, this trademark restriction does not form part of this License. - Creative Commons may be contacted at https://creativecommons.org/. \ No newline at end of file + Creative Commons may be contacted at https://creativecommons.org/. +-------------------------------------------------------------------------------- + +For: hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs +/server/datanode/checker/AbstractFuture.java and +hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs +/server/datanode/checker/TimeoutFuture.java + +Copyright (C) 2007 The Guava Authors + +Licensed under the Apache License, Version 2.0 (the "License"); you may not +use this file except in compliance with the License. You may obtain a copy of +the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations under +the License. diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml index 598f4fd2a65..be54efb2422 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml @@ -240,4 +240,16 @@ + + + + + + + + + + + + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/AbstractFuture.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/AbstractFuture.java new file mode 100644 index 00000000000..2e0ba18948a --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/AbstractFuture.java @@ -0,0 +1,1295 @@ +/* + * Copyright (C) 2007 The Guava Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +/** + * Some portions of this class have been modified to make it functional in this + * package. + */ +package org.apache.hadoop.hdfs.server.datanode.checker; + +import com.google.common.annotations.Beta; +import com.google.common.annotations.GwtCompatible; +import com.google.common.base.Preconditions; +import static com.google.common.base.Preconditions.checkNotNull; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.Uninterruptibles; +import static java.util.concurrent.atomic.AtomicReferenceFieldUpdater + .newUpdater; + +import javax.annotation.Nullable; +import java.security.AccessController; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; +import java.util.concurrent.locks.LockSupport; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * An abstract implementation of {@link ListenableFuture}, intended for + * advanced users only. More common ways to create a {@code ListenableFuture} + * include instantiating a {@link SettableFuture}, submitting a task to a + * {@link ListeningExecutorService}, and deriving a {@code Future} from an + * existing one, typically using methods like {@link Futures#transform + * (ListenableFuture, com.google.common.base.Function) Futures.transform} + * and {@link Futures#catching(ListenableFuture, Class, + * com.google.common.base.Function, java.util.concurrent.Executor) + * Futures.catching}. + *

    + *

    This class implements all methods in {@code ListenableFuture}. + * Subclasses should provide a way to set the result of the computation + * through the protected methods {@link #set(Object)}, + * {@link #setFuture(ListenableFuture)} and {@link #setException(Throwable)}. + * Subclasses may also override {@link #interruptTask()}, which will be + * invoked automatically if a call to {@link #cancel(boolean) cancel(true)} + * succeeds in canceling the future. Subclasses should rarely override other + * methods. + */ + +@GwtCompatible(emulated = true) +public abstract class AbstractFuture implements ListenableFuture { + // NOTE: Whenever both tests are cheap and functional, it's faster to use &, + // | instead of &&, || + + private static final boolean GENERATE_CANCELLATION_CAUSES = + Boolean.parseBoolean( + System.getProperty("guava.concurrent.generate_cancellation_cause", + "false")); + + /** + * A less abstract subclass of AbstractFuture. This can be used to optimize + * setFuture by ensuring that {@link #get} calls exactly the implementation + * of {@link AbstractFuture#get}. + */ + abstract static class TrustedFuture extends AbstractFuture { + @Override + public final V get() throws InterruptedException, ExecutionException { + return super.get(); + } + + @Override + public final V get(long timeout, TimeUnit unit) + throws InterruptedException, ExecutionException, TimeoutException { + return super.get(timeout, unit); + } + + @Override + public final boolean isDone() { + return super.isDone(); + } + + @Override + public final boolean isCancelled() { + return super.isCancelled(); + } + + @Override + public final void addListener(Runnable listener, Executor executor) { + super.addListener(listener, executor); + } + + @Override + public final boolean cancel(boolean mayInterruptIfRunning) { + return super.cancel(mayInterruptIfRunning); + } + } + + // Logger to log exceptions caught when running listeners. + private static final Logger log = Logger + .getLogger(AbstractFuture.class.getName()); + + // A heuristic for timed gets. If the remaining timeout is less than this, + // spin instead of + // blocking. This value is what AbstractQueuedSynchronizer uses. + private static final long SPIN_THRESHOLD_NANOS = 1000L; + + private static final AtomicHelper ATOMIC_HELPER; + + static { + AtomicHelper helper; + + try { + helper = new UnsafeAtomicHelper(); + } catch (Throwable unsafeFailure) { + // catch absolutely everything and fall through to our 'SafeAtomicHelper' + // The access control checks that ARFU does means the caller class has + // to be AbstractFuture + // instead of SafeAtomicHelper, so we annoyingly define these here + try { + helper = + new SafeAtomicHelper( + newUpdater(Waiter.class, Thread.class, "thread"), + newUpdater(Waiter.class, Waiter.class, "next"), + newUpdater(AbstractFuture.class, Waiter.class, "waiters"), + newUpdater(AbstractFuture.class, Listener.class, "listeners"), + newUpdater(AbstractFuture.class, Object.class, "value")); + } catch (Throwable atomicReferenceFieldUpdaterFailure) { + // Some Android 5.0.x Samsung devices have bugs in JDK reflection APIs + // that cause getDeclaredField to throw a NoSuchFieldException when + // the field is definitely there. + // For these users fallback to a suboptimal implementation, based on + // synchronized. This will be a definite performance hit to those users. + log.log(Level.SEVERE, "UnsafeAtomicHelper is broken!", unsafeFailure); + log.log( + Level.SEVERE, "SafeAtomicHelper is broken!", + atomicReferenceFieldUpdaterFailure); + helper = new SynchronizedHelper(); + } + } + ATOMIC_HELPER = helper; + + // Prevent rare disastrous classloading in first call to LockSupport.park. + // See: https://bugs.openjdk.java.net/browse/JDK-8074773 + @SuppressWarnings("unused") + Class ensureLoaded = LockSupport.class; + } + + /** + * Waiter links form a Treiber stack, in the {@link #waiters} field. + */ + private static final class Waiter { + static final Waiter TOMBSTONE = new Waiter(false /* ignored param */); + + @Nullable volatile Thread thread; + @Nullable volatile Waiter next; + + /** + * Constructor for the TOMBSTONE, avoids use of ATOMIC_HELPER in case this + * class is loaded before the ATOMIC_HELPER. Apparently this is possible + * on some android platforms. + */ + Waiter(boolean unused) { + } + + Waiter() { + // avoid volatile write, write is made visible by subsequent CAS on + // waiters field + ATOMIC_HELPER.putThread(this, Thread.currentThread()); + } + + // non-volatile write to the next field. Should be made visible by + // subsequent CAS on waiters field. + void setNext(Waiter next) { + ATOMIC_HELPER.putNext(this, next); + } + + void unpark() { + // This is racy with removeWaiter. The consequence of the race is that + // we may spuriously call unpark even though the thread has already + // removed itself from the list. But even if we did use a CAS, that + // race would still exist (it would just be ever so slightly smaller). + Thread w = thread; + if (w != null) { + thread = null; + LockSupport.unpark(w); + } + } + } + + /** + * Marks the given node as 'deleted' (null waiter) and then scans the list + * to unlink all deleted nodes. This is an O(n) operation in the common + * case (and O(n^2) in the worst), but we are saved by two things. + *

      + *
    • This is only called when a waiting thread times out or is + * interrupted. Both of which should be rare. + *
    • The waiters list should be very short. + *
    + */ + private void removeWaiter(Waiter node) { + node.thread = null; // mark as 'deleted' + restart: + while (true) { + Waiter pred = null; + Waiter curr = waiters; + if (curr == Waiter.TOMBSTONE) { + return; // give up if someone is calling complete + } + Waiter succ; + while (curr != null) { + succ = curr.next; + if (curr.thread != null) { // we aren't unlinking this node, update + // pred. + pred = curr; + } else if (pred != null) { // We are unlinking this node and it has a + // predecessor. + pred.next = succ; + if (pred.thread == null) { // We raced with another node that + // unlinked pred. Restart. + continue restart; + } + } else if (!ATOMIC_HELPER + .casWaiters(this, curr, succ)) { // We are unlinking head + continue restart; // We raced with an add or complete + } + curr = succ; + } + break; + } + } + + /** + * Listeners also form a stack through the {@link #listeners} field. + */ + private static final class Listener { + static final Listener TOMBSTONE = new Listener(null, null); + final Runnable task; + final Executor executor; + + // writes to next are made visible by subsequent CAS's on the listeners + // field + @Nullable Listener next; + + Listener(Runnable task, Executor executor) { + this.task = task; + this.executor = executor; + } + } + + /** + * A special value to represent {@code null}. + */ + private static final Object NULL = new Object(); + + /** + * A special value to represent failure, when {@link #setException} is + * called successfully. + */ + private static final class Failure { + static final Failure FALLBACK_INSTANCE = + new Failure( + new Throwable("Failure occurred while trying to finish a future" + + ".") { + @Override + public synchronized Throwable fillInStackTrace() { + return this; // no stack trace + } + }); + final Throwable exception; + + Failure(Throwable exception) { + this.exception = checkNotNull(exception); + } + } + + /** + * A special value to represent cancellation and the 'wasInterrupted' bit. + */ + private static final class Cancellation { + final boolean wasInterrupted; + @Nullable final Throwable cause; + + Cancellation(boolean wasInterrupted, @Nullable Throwable cause) { + this.wasInterrupted = wasInterrupted; + this.cause = cause; + } + } + + /** + * A special value that encodes the 'setFuture' state. + */ + private static final class SetFuture implements Runnable { + final AbstractFuture owner; + final ListenableFuture future; + + SetFuture(AbstractFuture owner, ListenableFuture future) { + this.owner = owner; + this.future = future; + } + + @Override + public void run() { + if (owner.value != this) { + // nothing to do, we must have been cancelled, don't bother inspecting + // the future. + return; + } + Object valueToSet = getFutureValue(future); + if (ATOMIC_HELPER.casValue(owner, this, valueToSet)) { + complete(owner); + } + } + } + + /** + * This field encodes the current state of the future. + *

    + *

    The valid values are: + *

      + *
    • {@code null} initial state, nothing has happened. + *
    • {@link Cancellation} terminal state, {@code cancel} was called. + *
    • {@link Failure} terminal state, {@code setException} was called. + *
    • {@link SetFuture} intermediate state, {@code setFuture} was called. + *
    • {@link #NULL} terminal state, {@code set(null)} was called. + *
    • Any other non-null value, terminal state, {@code set} was called with + * a non-null argument. + *
    + */ + private volatile Object value; + + /** + * All listeners. + */ + private volatile Listener listeners; + + /** + * All waiting threads. + */ + private volatile Waiter waiters; + + /** + * Constructor for use by subclasses. + */ + protected AbstractFuture() { + } + + // Gets and Timed Gets + // + // * Be responsive to interruption + // * Don't create Waiter nodes if you aren't going to park, this helps + // reduce contention on the waiters field. + // * Future completion is defined by when #value becomes non-null/non + // SetFuture + // * Future completion can be observed if the waiters field contains a + // TOMBSTONE + + // Timed Get + // There are a few design constraints to consider + // * We want to be responsive to small timeouts, unpark() has non trivial + // latency overheads (I have observed 12 micros on 64 bit linux systems to + // wake up a parked thread). So if the timeout is small we shouldn't park(). + // This needs to be traded off with the cpu overhead of spinning, so we use + // SPIN_THRESHOLD_NANOS which is what AbstractQueuedSynchronizer uses for + // similar purposes. + // * We want to behave reasonably for timeouts of 0 + // * We are more responsive to completion than timeouts. This is because + // parkNanos depends on system scheduling and as such we could either miss + // our deadline, or unpark() could be delayed so that it looks like we + // timed out even though we didn't. For comparison FutureTask respects + // completion preferably and AQS is non-deterministic (depends on where in + // the queue the waiter is). If we wanted to be strict about it, we could + // store the unpark() time in the Waiter node and we could use that to make + // a decision about whether or not we timed out prior to being unparked. + + /* + * Improve the documentation of when InterruptedException is thrown. Our + * behavior matches the JDK's, but the JDK's documentation is misleading. + */ + + /** + * {@inheritDoc} + *

    + *

    The default {@link AbstractFuture} implementation throws {@code + * InterruptedException} if the current thread is interrupted before or + * during the call, even if the value is already available. + * + * @throws InterruptedException if the current thread was interrupted + * before or during the call + * (optional but recommended). + * @throws CancellationException {@inheritDoc} + */ + @Override + public V get(long timeout, TimeUnit unit) + throws InterruptedException, TimeoutException, ExecutionException { + // NOTE: if timeout < 0, remainingNanos will be < 0 and we will fall into + // the while(true) loop at the bottom and throw a timeoutexception. + long remainingNanos = unit + .toNanos(timeout); // we rely on the implicit null check on unit. + if (Thread.interrupted()) { + throw new InterruptedException(); + } + Object localValue = value; + if (localValue != null & !(localValue instanceof SetFuture)) { + return getDoneValue(localValue); + } + // we delay calling nanoTime until we know we will need to either park or + // spin + final long endNanos = remainingNanos > 0 ? System + .nanoTime() + remainingNanos : 0; + long_wait_loop: + if (remainingNanos >= SPIN_THRESHOLD_NANOS) { + Waiter oldHead = waiters; + if (oldHead != Waiter.TOMBSTONE) { + Waiter node = new Waiter(); + do { + node.setNext(oldHead); + if (ATOMIC_HELPER.casWaiters(this, oldHead, node)) { + while (true) { + LockSupport.parkNanos(this, remainingNanos); + // Check interruption first, if we woke up due to interruption + // we need to honor that. + if (Thread.interrupted()) { + removeWaiter(node); + throw new InterruptedException(); + } + + // Otherwise re-read and check doneness. If we loop then it must + // have been a spurious wakeup + localValue = value; + if (localValue != null & !(localValue instanceof SetFuture)) { + return getDoneValue(localValue); + } + + // timed out? + remainingNanos = endNanos - System.nanoTime(); + if (remainingNanos < SPIN_THRESHOLD_NANOS) { + // Remove the waiter, one way or another we are done parking + // this thread. + removeWaiter(node); + break long_wait_loop; // jump down to the busy wait loop + } + } + } + oldHead = waiters; // re-read and loop. + } while (oldHead != Waiter.TOMBSTONE); + } + // re-read value, if we get here then we must have observed a TOMBSTONE + // while trying to add a waiter. + return getDoneValue(value); + } + // If we get here then we have remainingNanos < SPIN_THRESHOLD_NANOS and + // there is no node on the waiters list + while (remainingNanos > 0) { + localValue = value; + if (localValue != null & !(localValue instanceof SetFuture)) { + return getDoneValue(localValue); + } + if (Thread.interrupted()) { + throw new InterruptedException(); + } + remainingNanos = endNanos - System.nanoTime(); + } + throw new TimeoutException(); + } + + /* + * Improve the documentation of when InterruptedException is thrown. Our + * behavior matches the JDK's, but the JDK's documentation is misleading. + */ + + /** + * {@inheritDoc} + *

    + *

    The default {@link AbstractFuture} implementation throws {@code + * InterruptedException} if the current thread is interrupted before or + * during the call, even if the value is already available. + * + * @throws InterruptedException if the current thread was interrupted + * before or during the call + * (optional but recommended). + * @throws CancellationException {@inheritDoc} + */ + @Override + public V get() throws InterruptedException, ExecutionException { + if (Thread.interrupted()) { + throw new InterruptedException(); + } + Object localValue = value; + if (localValue != null & !(localValue instanceof SetFuture)) { + return getDoneValue(localValue); + } + Waiter oldHead = waiters; + if (oldHead != Waiter.TOMBSTONE) { + Waiter node = new Waiter(); + do { + node.setNext(oldHead); + if (ATOMIC_HELPER.casWaiters(this, oldHead, node)) { + // we are on the stack, now wait for completion. + while (true) { + LockSupport.park(this); + // Check interruption first, if we woke up due to interruption we + // need to honor that. + if (Thread.interrupted()) { + removeWaiter(node); + throw new InterruptedException(); + } + // Otherwise re-read and check doneness. If we loop then it must + // have been a spurious wakeup + localValue = value; + if (localValue != null & !(localValue instanceof SetFuture)) { + return getDoneValue(localValue); + } + } + } + oldHead = waiters; // re-read and loop. + } while (oldHead != Waiter.TOMBSTONE); + } + // re-read value, if we get here then we must have observed a TOMBSTONE + // while trying to add a waiter. + return getDoneValue(value); + } + + /** + * Unboxes {@code obj}. Assumes that obj is not {@code null} or a + * {@link SetFuture}. + */ + private V getDoneValue(Object obj) throws ExecutionException { + // While this seems like it might be too branch-y, simple benchmarking + // proves it to be unmeasurable (comparing done AbstractFutures with + // immediateFuture) + if (obj instanceof Cancellation) { + throw cancellationExceptionWithCause( + "Task was cancelled.", ((Cancellation) obj).cause); + } else if (obj instanceof Failure) { + throw new ExecutionException(((Failure) obj).exception); + } else if (obj == NULL) { + return null; + } else { + @SuppressWarnings("unchecked") // this is the only other option + V asV = (V) obj; + return asV; + } + } + + @Override + public boolean isDone() { + final Object localValue = value; + return localValue != null & !(localValue instanceof SetFuture); + } + + @Override + public boolean isCancelled() { + final Object localValue = value; + return localValue instanceof Cancellation; + } + + /** + * {@inheritDoc} + *

    + *

    If a cancellation attempt succeeds on a {@code Future} that had + * previously been {@linkplain#setFuture set asynchronously}, then the + * cancellation will also be propagated to the delegate {@code Future} that + * was supplied in the {@code setFuture} call. + */ + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + Object localValue = value; + boolean rValue = false; + if (localValue == null | localValue instanceof SetFuture) { + // Try to delay allocating the exception. At this point we may still + // lose the CAS, but it is certainly less likely. + Throwable cause = + GENERATE_CANCELLATION_CAUSES + ? new CancellationException("Future.cancel() was called.") + : null; + Object valueToSet = new Cancellation(mayInterruptIfRunning, cause); + AbstractFuture abstractFuture = this; + while (true) { + if (ATOMIC_HELPER.casValue(abstractFuture, localValue, valueToSet)) { + rValue = true; + // We call interuptTask before calling complete(), which is + // consistent with FutureTask + if (mayInterruptIfRunning) { + abstractFuture.interruptTask(); + } + complete(abstractFuture); + if (localValue instanceof SetFuture) { + // propagate cancellation to the future set in setfuture, this is + // racy, and we don't care if we are successful or not. + ListenableFuture futureToPropagateTo = ((SetFuture) localValue) + .future; + if (futureToPropagateTo instanceof TrustedFuture) { + // If the future is a TrustedFuture then we specifically avoid + // calling cancel() this has 2 benefits + // 1. for long chains of futures strung together with setFuture + // we consume less stack + // 2. we avoid allocating Cancellation objects at every level of + // the cancellation chain + // We can only do this for TrustedFuture, because + // TrustedFuture.cancel is final and does nothing but delegate + // to this method. + AbstractFuture trusted = (AbstractFuture) + futureToPropagateTo; + localValue = trusted.value; + if (localValue == null | localValue instanceof SetFuture) { + abstractFuture = trusted; + continue; // loop back up and try to complete the new future + } + } else { + // not a TrustedFuture, call cancel directly. + futureToPropagateTo.cancel(mayInterruptIfRunning); + } + } + break; + } + // obj changed, reread + localValue = abstractFuture.value; + if (!(localValue instanceof SetFuture)) { + // obj cannot be null at this point, because value can only change + // from null to non-null. So if value changed (and it did since we + // lost the CAS), then it cannot be null and since it isn't a + // SetFuture, then the future must be done and we should exit the loop + break; + } + } + } + return rValue; + } + + /** + * Subclasses can override this method to implement interruption of the + * future's computation. The method is invoked automatically by a + * successful call to {@link #cancel(boolean) cancel(true)}. + *

    + *

    The default implementation does nothing. + * + * @since 10.0 + */ + protected void interruptTask() { + } + + /** + * Returns true if this future was cancelled with {@code + * mayInterruptIfRunning} set to {@code true}. + * + * @since 14.0 + */ + protected final boolean wasInterrupted() { + final Object localValue = value; + return (localValue instanceof Cancellation) && ((Cancellation) localValue) + .wasInterrupted; + } + + /** + * {@inheritDoc} + * + * @since 10.0 + */ + @Override + public void addListener(Runnable listener, Executor executor) { + checkNotNull(listener, "Runnable was null."); + checkNotNull(executor, "Executor was null."); + Listener oldHead = listeners; + if (oldHead != Listener.TOMBSTONE) { + Listener newNode = new Listener(listener, executor); + do { + newNode.next = oldHead; + if (ATOMIC_HELPER.casListeners(this, oldHead, newNode)) { + return; + } + oldHead = listeners; // re-read + } while (oldHead != Listener.TOMBSTONE); + } + // If we get here then the Listener TOMBSTONE was set, which means the + // future is done, call the listener. + executeListener(listener, executor); + } + + /** + * Sets the result of this {@code Future} unless this {@code Future} has + * already been cancelled or set (including + * {@linkplain #setFuture set asynchronously}). When a call to this method + * returns, the {@code Future} is guaranteed to be + * {@linkplain #isDone done} only if the call was accepted (in which + * case it returns {@code true}). If it returns {@code false}, the {@code + * Future} may have previously been set asynchronously, in which case its + * result may not be known yet. That result, though not yet known, cannot + * be overridden by a call to a {@code set*} method, only by a call to + * {@link #cancel}. + * + * @param value the value to be used as the result + * @return true if the attempt was accepted, completing the {@code Future} + */ + protected boolean set(@Nullable V value) { + Object valueToSet = value == null ? NULL : value; + if (ATOMIC_HELPER.casValue(this, null, valueToSet)) { + complete(this); + return true; + } + return false; + } + + /** + * Sets the failed result of this {@code Future} unless this {@code Future} + * has already been cancelled or set (including + * {@linkplain #setFuture set asynchronously}). When a call to this method + * returns, the {@code Future} is guaranteed to be + * {@linkplain #isDone done} only if the call was accepted (in which + * case it returns {@code true}). If it returns {@code false}, the + * {@code Future} may have previously been set asynchronously, in which case + * its result may not be known yet. That result, though not yet known, + * cannot be overridden by a call to a {@code set*} method, only by a call + * to {@link #cancel}. + * + * @param throwable the exception to be used as the failed result + * @return true if the attempt was accepted, completing the {@code Future} + */ + protected boolean setException(Throwable throwable) { + Object valueToSet = new Failure(checkNotNull(throwable)); + if (ATOMIC_HELPER.casValue(this, null, valueToSet)) { + complete(this); + return true; + } + return false; + } + + /** + * Sets the result of this {@code Future} to match the supplied input + * {@code Future} once the supplied {@code Future} is done, unless this + * {@code Future} has already been cancelled or set (including "set + * asynchronously," defined below). + *

    + *

    If the supplied future is {@linkplain #isDone done} when this method + * is called and the call is accepted, then this future is guaranteed to + * have been completed with the supplied future by the time this method + * returns. If the supplied future is not done and the call is accepted, then + * the future will be set asynchronously. Note that such a result, + * though not yet known, cannot be overridden by a call to a {@code set*} + * method, only by a call to {@link #cancel}. + *

    + *

    If the call {@code setFuture(delegate)} is accepted and this {@code + * Future} is later cancelled, cancellation will be propagated to {@code + * delegate}. Additionally, any call to {@code setFuture} after any + * cancellation will propagate cancellation to the supplied {@code Future}. + * + * @param future the future to delegate to + * @return true if the attempt was accepted, indicating that the {@code + * Future} was not previously cancelled or set. + * @since 19.0 + */ + @Beta + protected boolean setFuture(ListenableFuture future) { + checkNotNull(future); + Object localValue = value; + if (localValue == null) { + if (future.isDone()) { + Object value = getFutureValue(future); + if (ATOMIC_HELPER.casValue(this, null, value)) { + complete(this); + return true; + } + return false; + } + SetFuture valueToSet = new SetFuture(this, future); + if (ATOMIC_HELPER.casValue(this, null, valueToSet)) { + // the listener is responsible for calling completeWithFuture, + // directExecutor is appropriate since all we are doing is unpacking + // a completed future which should be fast. + try { + future.addListener(valueToSet, directExecutor()); + } catch (Throwable t) { + // addListener has thrown an exception! SetFuture.run can't throw + // any exceptions so this must have been caused by addListener + // itself. The most likely explanation is a misconfigured mock. Try + // to switch to Failure. + Failure failure; + try { + failure = new Failure(t); + } catch (Throwable oomMostLikely) { + failure = Failure.FALLBACK_INSTANCE; + } + // Note: The only way this CAS could fail is if cancel() has raced + // with us. That is ok. + boolean unused = ATOMIC_HELPER.casValue(this, valueToSet, failure); + } + return true; + } + localValue = value; // we lost the cas, fall through and maybe cancel + } + // The future has already been set to something. If it is cancellation we + // should cancel the incoming future. + if (localValue instanceof Cancellation) { + // we don't care if it fails, this is best-effort. + future.cancel(((Cancellation) localValue).wasInterrupted); + } + return false; + } + + /** + * Returns a value, suitable for storing in the {@link #value} field. From + * the given future, which is assumed to be done. + *

    + *

    This is approximately the inverse of {@link #getDoneValue(Object)} + */ + private static Object getFutureValue(ListenableFuture future) { + Object valueToSet; + if (future instanceof TrustedFuture) { + // Break encapsulation for TrustedFuture instances since we know that + // subclasses cannot override .get() (since it is final) and therefore + // this is equivalent to calling .get() and unpacking the exceptions + // like we do below (just much faster because it is a single field read + // instead of a read, several branches and possibly creating exceptions). + return ((AbstractFuture) future).value; + } else { + // Otherwise calculate valueToSet by calling .get() + try { + Object v = getDone(future); + valueToSet = v == null ? NULL : v; + } catch (ExecutionException exception) { + valueToSet = new Failure(exception.getCause()); + } catch (CancellationException cancellation) { + valueToSet = new Cancellation(false, cancellation); + } catch (Throwable t) { + valueToSet = new Failure(t); + } + } + return valueToSet; + } + + /** + * Unblocks all threads and runs all listeners. + */ + private static void complete(AbstractFuture future) { + Listener next = null; + outer: + while (true) { + future.releaseWaiters(); + // We call this before the listeners in order to avoid needing to manage + // a separate stack data structure for them. afterDone() should be + // generally fast and only used for cleanup work... but in theory can + // also be recursive and create StackOverflowErrors + future.afterDone(); + // push the current set of listeners onto next + next = future.clearListeners(next); + future = null; + while (next != null) { + Listener curr = next; + next = next.next; + Runnable task = curr.task; + if (task instanceof SetFuture) { + SetFuture setFuture = (SetFuture) task; + // We unwind setFuture specifically to avoid StackOverflowErrors in + // the case of long chains of SetFutures + // Handling this special case is important because there is no way + // to pass an executor to setFuture, so a user couldn't break the + // chain by doing this themselves. It is also potentially common + // if someone writes a recursive Futures.transformAsync transformer. + future = setFuture.owner; + if (future.value == setFuture) { + Object valueToSet = getFutureValue(setFuture.future); + if (ATOMIC_HELPER.casValue(future, setFuture, valueToSet)) { + continue outer; + } + } + // other wise the future we were trying to set is already done. + } else { + executeListener(task, curr.executor); + } + } + break; + } + } + + public static V getDone(Future future) throws ExecutionException { + /* + * We throw IllegalStateException, since the call could succeed later. + * Perhaps we "should" throw IllegalArgumentException, since the call + * could succeed with a different argument. Those exceptions' docs + * suggest that either is acceptable. Google's Java Practices page + * recommends IllegalArgumentException here, in part to keep its + * recommendation simple: Static methods should throw + * IllegalStateException only when they use static state. + * + * + * Why do we deviate here? The answer: We want for fluentFuture.getDone() + * to throw the same exception as Futures.getDone(fluentFuture). + */ + Preconditions.checkState(future.isDone(), "Future was expected to be " + + "done:" + + " %s", future); + return Uninterruptibles.getUninterruptibly(future); + } + + /** + * Callback method that is called exactly once after the future is completed. + *

    + *

    If {@link #interruptTask} is also run during completion, + * {@link #afterDone} runs after it. + *

    + *

    The default implementation of this method in {@code AbstractFuture} + * does nothing. This is intended for very lightweight cleanup work, for + * example, timing statistics or clearing fields. + * If your task does anything heavier consider, just using a listener with + * an executor. + * + * @since 20.0 + */ + @Beta + protected void afterDone() { + } + + /** + * If this future has been cancelled (and possibly interrupted), cancels + * (and possibly interrupts) the given future (if available). + *

    + *

    This method should be used only when this future is completed. It is + * designed to be called from {@code done}. + */ + final void maybePropagateCancellation(@Nullable Future related) { + if (related != null & isCancelled()) { + related.cancel(wasInterrupted()); + } + } + + /** + * Releases all threads in the {@link #waiters} list, and clears the list. + */ + private void releaseWaiters() { + Waiter head; + do { + head = waiters; + } while (!ATOMIC_HELPER.casWaiters(this, head, Waiter.TOMBSTONE)); + for ( + Waiter currentWaiter = head; + currentWaiter != null; + currentWaiter = currentWaiter.next) { + currentWaiter.unpark(); + } + } + + /** + * Clears the {@link #listeners} list and prepends its contents to {@code + * onto}, least recently added first. + */ + private Listener clearListeners(Listener onto) { + // We need to + // 1. atomically swap the listeners with TOMBSTONE, this is because + // addListener uses that to to synchronize with us + // 2. reverse the linked list, because despite our rather clear contract, + // people depend on us executing listeners in the order they were added + // 3. push all the items onto 'onto' and return the new head of the stack + Listener head; + do { + head = listeners; + } while (!ATOMIC_HELPER.casListeners(this, head, Listener.TOMBSTONE)); + Listener reversedList = onto; + while (head != null) { + Listener tmp = head; + head = head.next; + tmp.next = reversedList; + reversedList = tmp; + } + return reversedList; + } + + /** + * Submits the given runnable to the given {@link Executor} catching and + * logging all {@linkplain RuntimeException runtime exceptions} thrown by + * the executor. + */ + private static void executeListener(Runnable runnable, Executor executor) { + try { + executor.execute(runnable); + } catch (RuntimeException e) { + // Log it and keep going -- bad runnable and/or executor. Don't punish + // the other runnables if we're given a bad one. We only catch + // RuntimeException because we want Errors to propagate up. + log.log( + Level.SEVERE, + "RuntimeException while executing runnable " + runnable + " with " + + "executor " + executor, + e); + } + } + + private abstract static class AtomicHelper { + /** + * Non volatile write of the thread to the {@link Waiter#thread} field. + */ + abstract void putThread(Waiter waiter, Thread newValue); + + /** + * Non volatile write of the waiter to the {@link Waiter#next} field. + */ + abstract void putNext(Waiter waiter, Waiter newValue); + + /** + * Performs a CAS operation on the {@link #waiters} field. + */ + abstract boolean casWaiters( + AbstractFuture future, Waiter expect, + Waiter update); + + /** + * Performs a CAS operation on the {@link #listeners} field. + */ + abstract boolean casListeners( + AbstractFuture future, Listener expect, + Listener update); + + /** + * Performs a CAS operation on the {@link #value} field. + */ + abstract boolean casValue( + AbstractFuture future, Object expect, Object update); + } + + /** + * {@link AtomicHelper} based on {@link sun.misc.Unsafe}. + *

    + *

    Static initialization of this class will fail if the + * {@link sun.misc.Unsafe} object cannot be accessed. + */ + private static final class UnsafeAtomicHelper extends AtomicHelper { + static final sun.misc.Unsafe UNSAFE; + static final long LISTENERS_OFFSET; + static final long WAITERS_OFFSET; + static final long VALUE_OFFSET; + static final long WAITER_THREAD_OFFSET; + static final long WAITER_NEXT_OFFSET; + + static { + sun.misc.Unsafe unsafe = null; + try { + unsafe = sun.misc.Unsafe.getUnsafe(); + } catch (SecurityException tryReflectionInstead) { + try { + unsafe = + AccessController.doPrivileged( + new PrivilegedExceptionAction() { + @Override + public sun.misc.Unsafe run() throws Exception { + Class k = sun.misc.Unsafe.class; + for (java.lang.reflect.Field f : k.getDeclaredFields()) { + f.setAccessible(true); + Object x = f.get(null); + if (k.isInstance(x)) { + return k.cast(x); + } + } + throw new NoSuchFieldError("the Unsafe"); + } + }); + } catch (PrivilegedActionException e) { + throw new RuntimeException( + "Could not initialize intrinsics", e.getCause()); + } + } + try { + Class abstractFuture = AbstractFuture.class; + WAITERS_OFFSET = unsafe + .objectFieldOffset(abstractFuture.getDeclaredField("waiters")); + LISTENERS_OFFSET = unsafe + .objectFieldOffset(abstractFuture.getDeclaredField("listeners")); + VALUE_OFFSET = unsafe + .objectFieldOffset(abstractFuture.getDeclaredField("value")); + WAITER_THREAD_OFFSET = unsafe + .objectFieldOffset(Waiter.class.getDeclaredField("thread")); + WAITER_NEXT_OFFSET = unsafe + .objectFieldOffset(Waiter.class.getDeclaredField("next")); + UNSAFE = unsafe; + } catch (Exception e) { + throwIfUnchecked(e); + throw new RuntimeException(e); + } + } + + public static void throwIfUnchecked(Throwable throwable) { + checkNotNull(throwable); + if (throwable instanceof RuntimeException) { + throw (RuntimeException) throwable; + } + if (throwable instanceof Error) { + throw (Error) throwable; + } + } + + @Override + void putThread(Waiter waiter, Thread newValue) { + UNSAFE.putObject(waiter, WAITER_THREAD_OFFSET, newValue); + } + + @Override + void putNext(Waiter waiter, Waiter newValue) { + UNSAFE.putObject(waiter, WAITER_NEXT_OFFSET, newValue); + } + + /** + * Performs a CAS operation on the {@link #waiters} field. + */ + @Override + boolean casWaiters(AbstractFuture future, Waiter expect, Waiter + update) { + return UNSAFE + .compareAndSwapObject(future, WAITERS_OFFSET, expect, update); + } + + /** + * Performs a CAS operation on the {@link #listeners} field. + */ + @Override + boolean casListeners( + AbstractFuture future, Listener expect, Listener update) { + return UNSAFE + .compareAndSwapObject(future, LISTENERS_OFFSET, expect, update); + } + + /** + * Performs a CAS operation on the {@link #value} field. + */ + @Override + boolean casValue(AbstractFuture future, Object expect, Object update) { + return UNSAFE.compareAndSwapObject(future, VALUE_OFFSET, expect, update); + } + } + + /** + * {@link AtomicHelper} based on {@link AtomicReferenceFieldUpdater}. + */ + private static final class SafeAtomicHelper extends AtomicHelper { + final AtomicReferenceFieldUpdater waiterThreadUpdater; + final AtomicReferenceFieldUpdater waiterNextUpdater; + final AtomicReferenceFieldUpdater waitersUpdater; + final AtomicReferenceFieldUpdater + listenersUpdater; + final AtomicReferenceFieldUpdater valueUpdater; + + SafeAtomicHelper( + AtomicReferenceFieldUpdater waiterThreadUpdater, + AtomicReferenceFieldUpdater waiterNextUpdater, + AtomicReferenceFieldUpdater waitersUpdater, + AtomicReferenceFieldUpdater listenersUpdater, + AtomicReferenceFieldUpdater valueUpdater) { + this.waiterThreadUpdater = waiterThreadUpdater; + this.waiterNextUpdater = waiterNextUpdater; + this.waitersUpdater = waitersUpdater; + this.listenersUpdater = listenersUpdater; + this.valueUpdater = valueUpdater; + } + + @Override + void putThread(Waiter waiter, Thread newValue) { + waiterThreadUpdater.lazySet(waiter, newValue); + } + + @Override + void putNext(Waiter waiter, Waiter newValue) { + waiterNextUpdater.lazySet(waiter, newValue); + } + + @Override + boolean casWaiters(AbstractFuture future, Waiter expect, Waiter + update) { + return waitersUpdater.compareAndSet(future, expect, update); + } + + @Override + boolean casListeners( + AbstractFuture future, Listener expect, Listener update) { + return listenersUpdater.compareAndSet(future, expect, update); + } + + @Override + boolean casValue(AbstractFuture future, Object expect, Object update) { + return valueUpdater.compareAndSet(future, expect, update); + } + } + + /** + * {@link AtomicHelper} based on {@code synchronized} and volatile writes. + *

    + *

    This is an implementation of last resort for when certain basic VM + * features are broken (like AtomicReferenceFieldUpdater). + */ + private static final class SynchronizedHelper extends AtomicHelper { + @Override + void putThread(Waiter waiter, Thread newValue) { + waiter.thread = newValue; + } + + @Override + void putNext(Waiter waiter, Waiter newValue) { + waiter.next = newValue; + } + + @Override + boolean casWaiters(AbstractFuture future, Waiter expect, Waiter + update) { + synchronized (future) { + if (future.waiters == expect) { + future.waiters = update; + return true; + } + return false; + } + } + + @Override + boolean casListeners( + AbstractFuture future, Listener expect, Listener update) { + synchronized (future) { + if (future.listeners == expect) { + future.listeners = update; + return true; + } + return false; + } + } + + @Override + boolean casValue(AbstractFuture future, Object expect, Object update) { + synchronized (future) { + if (future.value == expect) { + future.value = update; + return true; + } + return false; + } + } + } + + private static CancellationException cancellationExceptionWithCause( + @Nullable String message, @Nullable Throwable cause) { + CancellationException exception = new CancellationException(message); + exception.initCause(cause); + return exception; + } + + /** + * Returns an {@link Executor} that runs each task in the thread that invokes + * {@link Executor#execute execute}, as in {@link CallerRunsPolicy}. + *

    + *

    This instance is equivalent to: 

       {@code
+   *   final class DirectExecutor implements Executor {
+   *     public void execute(Runnable r) {
+   *       r.run();
+   *     }
+   *   }}
    + *

    + *

    This should be preferred to {@link #newDirectExecutorService()} + * because implementing the {@link ExecutorService} subinterface + * necessitates significant performance overhead. + * + * @since 18.0 + */ + public static Executor directExecutor() { + return DirectExecutor.INSTANCE; + } + + /** + * See {@link #directExecutor} for behavioral notes. + */ + private enum DirectExecutor implements Executor { + INSTANCE; + + @Override + public void execute(Runnable command) { + command.run(); + } + + @Override + public String toString() { + return "MoreExecutors.directExecutor()"; + } + } + +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java index 9ad47f012d7..6ab6425ad79 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java @@ -91,6 +91,7 @@ public class DatasetVolumeChecker { * Minimum time between two successive disk checks of a volume. */ private final long minDiskCheckGapMs; + private final long diskCheckTimeout; /** * Timestamp of the last check of all volumes. @@ -136,6 +137,17 @@ public class DatasetVolumeChecker { + minDiskCheckGapMs + " (should be >= 0)"); } + diskCheckTimeout = conf.getTimeDuration( + DFSConfigKeys.DFS_DATANODE_DISK_CHECK_TIMEOUT_KEY, + DFSConfigKeys.DFS_DATANODE_DISK_CHECK_TIMEOUT_DEFAULT, + TimeUnit.MILLISECONDS); + + if (diskCheckTimeout < 0) { + throw new DiskErrorException("Invalid value configured for " + + DFS_DATANODE_DISK_CHECK_TIMEOUT_KEY + " - " + + diskCheckTimeout + " (should be >= 0)"); + } + lastAllVolumesCheck = timer.monotonicNow() - minDiskCheckGapMs; if (maxVolumeFailuresTolerated < 0) { @@ -145,7 +157,8 @@ public class DatasetVolumeChecker { } delegateChecker = new ThrottledAsyncChecker<>( - timer, minDiskCheckGapMs, Executors.newCachedThreadPool( + timer, minDiskCheckGapMs, diskCheckTimeout, + Executors.newCachedThreadPool( new ThreadFactoryBuilder() .setNameFormat("DataNode DiskChecker thread %d") .setDaemon(true) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/StorageLocationChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/StorageLocationChecker.java index a0bffcd01e6..2d1eebe0bd6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/StorageLocationChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/StorageLocationChecker.java @@ -119,6 +119,7 @@ public class StorageLocationChecker { DFSConfigKeys.DFS_DATANODE_DISK_CHECK_MIN_GAP_KEY, DFSConfigKeys.DFS_DATANODE_DISK_CHECK_MIN_GAP_DEFAULT, TimeUnit.MILLISECONDS), + 0, Executors.newCachedThreadPool( new ThreadFactoryBuilder() .setNameFormat("StorageLocationChecker thread %d") diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/ThrottledAsyncChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/ThrottledAsyncChecker.java index 83c554d4dc5..7584d97b5f2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/ThrottledAsyncChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/ThrottledAsyncChecker.java @@ -38,6 +38,8 @@ import java.util.Map; import java.util.WeakHashMap; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; /** @@ -64,12 +66,14 @@ public class ThrottledAsyncChecker implements AsyncChecker { * The ExecutorService used to schedule asynchronous checks. */ private final ListeningExecutorService executorService; + private final ScheduledExecutorService scheduledExecutorService; /** * The minimum gap in milliseconds between two successive checks * of the same object. This is the throttle. */ private final long minMsBetweenChecks; + private final long diskCheckTimeout; /** * Map of checks that are currently in progress. Protected by the object @@ -86,12 +90,23 @@ public class ThrottledAsyncChecker implements AsyncChecker { ThrottledAsyncChecker(final Timer timer, final long minMsBetweenChecks, + final long diskCheckTimeout, final ExecutorService executorService) { this.timer = timer; this.minMsBetweenChecks = minMsBetweenChecks; + this.diskCheckTimeout = diskCheckTimeout; this.executorService = MoreExecutors.listeningDecorator(executorService); this.checksInProgress = new HashMap<>(); this.completedChecks = new WeakHashMap<>(); + + if (this.diskCheckTimeout > 0) { + ScheduledThreadPoolExecutor scheduledThreadPoolExecutor = new + ScheduledThreadPoolExecutor(1); + this.scheduledExecutorService = MoreExecutors + .getExitingScheduledExecutorService(scheduledThreadPoolExecutor); + } else { + this.scheduledExecutorService = null; + } } /** @@ -120,13 +135,23 @@ public class ThrottledAsyncChecker implements AsyncChecker { } } - final ListenableFuture lf = executorService.submit( + final ListenableFuture lfWithoutTimeout = executorService.submit( new Callable() { @Override public V call() throws Exception { return target.check(context); } }); + final ListenableFuture lf; + + if (diskCheckTimeout > 0) { + lf = TimeoutFuture + .create(lfWithoutTimeout, diskCheckTimeout, TimeUnit.MILLISECONDS, + scheduledExecutorService); + } else { + lf = lfWithoutTimeout; + } + checksInProgress.put(target, lf); addResultCachingCallback(target, lf); return Optional.of(lf); @@ -174,6 +199,16 @@ public class ThrottledAsyncChecker implements AsyncChecker { executorService.shutdownNow(); executorService.awaitTermination(timeout, timeUnit); } + if (scheduledExecutorService != null) { + // Try orderly shutdown + scheduledExecutorService.shutdown(); + + if (!scheduledExecutorService.awaitTermination(timeout, timeUnit)) { + // Interrupt executing tasks and wait again. + scheduledExecutorService.shutdownNow(); + scheduledExecutorService.awaitTermination(timeout, timeUnit); + } + } } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/TimeoutFuture.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/TimeoutFuture.java new file mode 100644 index 00000000000..ae7b34f7734 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/TimeoutFuture.java @@ -0,0 +1,162 @@ +/* + * Copyright (C) 2007 The Guava Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +/** + * Some portions of this class have been modified to make it functional in this + * package. + */ +package org.apache.hadoop.hdfs.server.datanode.checker; + +import com.google.common.base.Preconditions; +import com.google.common.util.concurrent.ListenableFuture; +import org.apache.hadoop.hdfs.server.datanode.checker.AbstractFuture; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** + * Implementation of {@code Futures#withTimeout}. + *

    + *

    Future that delegates to another but will finish early (via a + * {@link TimeoutException} wrapped in an {@link ExecutionException}) if the + * specified duration expires. The delegate future is interrupted and + * cancelled if it times out. + */ +final class TimeoutFuture extends AbstractFuture.TrustedFuture { + public static final Logger LOG = LoggerFactory.getLogger( + TimeoutFuture.class); + + static ListenableFuture create( + ListenableFuture delegate, + long time, + TimeUnit unit, + ScheduledExecutorService scheduledExecutor) { + TimeoutFuture result = new TimeoutFuture(delegate); + TimeoutFuture.Fire fire = new TimeoutFuture.Fire(result); + result.timer = scheduledExecutor.schedule(fire, time, unit); + delegate.addListener(fire, directExecutor()); + return result; + } + + /* + * Memory visibility of these fields. There are two cases to consider. + * + * 1. visibility of the writes to these fields to Fire.run: + * + * The initial write to delegateRef is made definitely visible via the + * semantics of addListener/SES.schedule. The later racy write in cancel() + * is not guaranteed to be observed, however that is fine since the + * correctness is based on the atomic state in our base class. The initial + * write to timer is never definitely visible to Fire.run since it is + * assigned after SES.schedule is called. Therefore Fire.run has to check + * for null. However, it should be visible if Fire.run is called by + * delegate.addListener since addListener is called after the assignment + * to timer, and importantly this is the main situation in which we need to + * be able to see the write. + * + * 2. visibility of the writes to an afterDone() call triggered by cancel(): + * + * Since these fields are non-final that means that TimeoutFuture is not + * being 'safely published', thus a motivated caller may be able to expose + * the reference to another thread that would then call cancel() and be + * unable to cancel the delegate. There are a number of ways to solve this, + * none of which are very pretty, and it is currently believed to be a + * purely theoretical problem (since the other actions should supply + * sufficient write-barriers). + */ + + @Nullable private ListenableFuture delegateRef; + @Nullable private Future timer; + + private TimeoutFuture(ListenableFuture delegate) { + this.delegateRef = Preconditions.checkNotNull(delegate); + } + + /** + * A runnable that is called when the delegate or the timer completes. + */ + private static final class Fire implements Runnable { + @Nullable + TimeoutFuture timeoutFutureRef; + + Fire( + TimeoutFuture timeoutFuture) { + this.timeoutFutureRef = timeoutFuture; + } + + @Override + public void run() { + // If either of these reads return null then we must be after a + // successful cancel or another call to this method. + TimeoutFuture timeoutFuture = timeoutFutureRef; + if (timeoutFuture == null) { + return; + } + ListenableFuture delegate = timeoutFuture.delegateRef; + if (delegate == null) { + return; + } + + /* + * If we're about to complete the TimeoutFuture, we want to release our + * reference to it. Otherwise, we'll pin it (and its result) in memory + * until the timeout task is GCed. (The need to clear our reference to + * the TimeoutFuture is the reason we use a *static* nested class with + * a manual reference back to the "containing" class.) + * + * This has the nice-ish side effect of limiting reentrancy: run() calls + * timeoutFuture.setException() calls run(). That reentrancy would + * already be harmless, since timeoutFuture can be set (and delegate + * cancelled) only once. (And "set only once" is important for other + * reasons: run() can still be invoked concurrently in different threads, + * even with the above null checks.) + */ + timeoutFutureRef = null; + if (delegate.isDone()) { + timeoutFuture.setFuture(delegate); + } else { + try { + timeoutFuture.setException( + new TimeoutException("Future timed out: " + delegate)); + } finally { + delegate.cancel(true); + } + } + } + } + + @Override + protected void afterDone() { + maybePropagateCancellation(delegateRef); + + Future localTimer = timer; + // Try to cancel the timer as an optimization. + // timer may be null if this call to run was by the timer task since there + // is no happens-before edge between the assignment to timer and an + // execution of the timer task. + if (localTimer != null) { + localTimer.cancel(false); + } + + delegateRef = null; + timer = null; + } +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestDatasetVolumeCheckerTimeout.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestDatasetVolumeCheckerTimeout.java new file mode 100644 index 00000000000..dc091ed31b9 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestDatasetVolumeCheckerTimeout.java @@ -0,0 +1,134 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.datanode.checker; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.LogVerificationAppender; +import org.apache.hadoop.hdfs.server.datanode.StorageLocation; +import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference; +import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; +import org.apache.hadoop.util.FakeTimer; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.apache.log4j.spi.LoggingEvent; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; +import static org.mockito.Matchers.anyObject; +import static org.mockito.Mockito.*; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.ReentrantLock; + +/** + * Test that timeout is triggered during Disk Volume Checker. + */ +public class TestDatasetVolumeCheckerTimeout { + public static final org.slf4j.Logger LOG = + LoggerFactory.getLogger(TestDatasetVolumeCheckerTimeout.class); + + @Rule + public TestName testName = new TestName(); + + static Configuration conf; + private static final long DISK_CHECK_TIMEOUT = 10; + private static final long DISK_CHECK_TIME = 100; + static ReentrantLock lock = new ReentrantLock(); + + static { + conf = new HdfsConfiguration(); + conf.setTimeDuration( + DFSConfigKeys.DFS_DATANODE_DISK_CHECK_TIMEOUT_KEY, + DISK_CHECK_TIMEOUT, TimeUnit.MILLISECONDS); + } + + static FsVolumeSpi makeSlowVolume() throws Exception { + final FsVolumeSpi volume = mock(FsVolumeSpi.class); + final FsVolumeReference reference = mock(FsVolumeReference.class); + final StorageLocation location = mock(StorageLocation.class); + + when(reference.getVolume()).thenReturn(volume); + when(volume.obtainReference()).thenReturn(reference); + when(volume.getStorageLocation()).thenReturn(location); + + when(volume.check(anyObject())).thenAnswer(new Answer() { + @Override + public VolumeCheckResult answer( + InvocationOnMock invocationOnMock) throws Throwable { + // Wait for the disk check to timeout and then release lock. + lock.lock(); + lock.unlock(); + return VolumeCheckResult.HEALTHY; + } + }); + + return volume; + } + + @Test (timeout = 1000) + public void testDiskCheckTimeout() throws Exception { + LOG.info("Executing {}", testName.getMethodName()); + final FsVolumeSpi volume = makeSlowVolume(); + + final DatasetVolumeChecker checker = + new DatasetVolumeChecker(conf, new FakeTimer()); + final AtomicLong numCallbackInvocations = new AtomicLong(0); + + lock.lock(); + /** + * Request a check and ensure it triggered {@link FsVolumeSpi#check}. + */ + boolean result = + checker.checkVolume(volume, new DatasetVolumeChecker.Callback() { + @Override + public void call(Set healthyVolumes, + Set failedVolumes) { + numCallbackInvocations.incrementAndGet(); + + // Assert that the disk check registers a failed volume due to + // timeout + assertThat(healthyVolumes.size(), is(0)); + assertThat(failedVolumes.size(), is(1)); + } + }); + + // Wait for the callback + Thread.sleep(DISK_CHECK_TIME); + + // Release lock + lock.unlock(); + + // Ensure that the check was invoked only once. + verify(volume, times(1)).check(anyObject()); + assertThat(numCallbackInvocations.get(), is(1L)); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestThrottledAsyncChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestThrottledAsyncChecker.java index c171c0fb257..00b1af2a11a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestThrottledAsyncChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestThrottledAsyncChecker.java @@ -59,7 +59,7 @@ public class TestThrottledAsyncChecker { final NoOpCheckable target2 = new NoOpCheckable(); final FakeTimer timer = new FakeTimer(); ThrottledAsyncChecker checker = - new ThrottledAsyncChecker<>(timer, MIN_ERROR_CHECK_GAP, + new ThrottledAsyncChecker<>(timer, MIN_ERROR_CHECK_GAP, 0, getExecutorService()); // check target1 and ensure we get back the expected result. @@ -99,8 +99,8 @@ public class TestThrottledAsyncChecker { final FakeTimer timer = new FakeTimer(); final LatchedCallback callback = new LatchedCallback(target); ThrottledAsyncChecker checker = - new ThrottledAsyncChecker<>(timer, MIN_ERROR_CHECK_GAP, - getExecutorService()); + new ThrottledAsyncChecker<>(timer, MIN_ERROR_CHECK_GAP, 0, + getExecutorService()); Optional> olf = checker.schedule(target, true); @@ -124,8 +124,8 @@ public class TestThrottledAsyncChecker { LatchedCheckable target = new LatchedCheckable(); final FakeTimer timer = new FakeTimer(); ThrottledAsyncChecker checker = - new ThrottledAsyncChecker<>(timer, MIN_ERROR_CHECK_GAP, - getExecutorService()); + new ThrottledAsyncChecker<>(timer, MIN_ERROR_CHECK_GAP, 0, + getExecutorService()); final Optional> olf1 = checker.schedule(target, true); @@ -167,7 +167,7 @@ public class TestThrottledAsyncChecker { final NoOpCheckable target1 = new NoOpCheckable(); final FakeTimer timer = new FakeTimer(); ThrottledAsyncChecker checker = - new ThrottledAsyncChecker<>(timer, MIN_ERROR_CHECK_GAP, + new ThrottledAsyncChecker<>(timer, MIN_ERROR_CHECK_GAP, 0, getExecutorService()); assertTrue(checker.schedule(target1, true).isPresent()); @@ -203,7 +203,7 @@ public class TestThrottledAsyncChecker { final ThrowingCheckable target1 = new ThrowingCheckable(); final FakeTimer timer = new FakeTimer(); ThrottledAsyncChecker checker = - new ThrottledAsyncChecker<>(timer, MIN_ERROR_CHECK_GAP, + new ThrottledAsyncChecker<>(timer, MIN_ERROR_CHECK_GAP, 0, getExecutorService()); assertTrue(checker.schedule(target1, true).isPresent()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestThrottledAsyncCheckerTimeout.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestThrottledAsyncCheckerTimeout.java new file mode 100644 index 00000000000..52cab57dc6f --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/checker/TestThrottledAsyncCheckerTimeout.java @@ -0,0 +1,223 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.datanode.checker; + +import com.google.common.base.Optional; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.LogVerificationAppender; +import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; +import org.apache.hadoop.util.FakeTimer; +import org.apache.log4j.Logger; +import org.apache.log4j.spi.LoggingEvent; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.*; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyObject; +import static org.mockito.Matchers.anySet; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import org.slf4j.LoggerFactory; + +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.ReentrantLock; + +public class TestThrottledAsyncCheckerTimeout { + public static final org.slf4j.Logger LOG = + LoggerFactory.getLogger(TestThrottledAsyncCheckerTimeout.class); + + @Rule + public TestName testName = new TestName(); + + Configuration conf; + private static final long DISK_CHECK_TIMEOUT = 10; + private static final long DISK_CHECK_TIME = 100; + private ReentrantLock lock; + + private ExecutorService getExecutorService() { + return new ScheduledThreadPoolExecutor(1); + } + + @Before + public void initializeLock() { + lock = new ReentrantLock(); + } + + @Test (timeout = 1000) + public void testDiskCheckTimeout() throws Exception { + LOG.info("Executing {}", testName.getMethodName()); + + final DummyCheckable target = new DummyCheckable(); + final FakeTimer timer = new FakeTimer(); + ThrottledAsyncChecker checker = + new ThrottledAsyncChecker<>(timer, 0, DISK_CHECK_TIMEOUT, + getExecutorService()); + + // Acquire lock to halt checker. Release after timeout occurs. + lock.lock(); + + final Optional> olf = checker + .schedule(target, true); + + final AtomicLong numCallbackInvocationsSuccess = new AtomicLong(0); + final AtomicLong numCallbackInvocationsFailure = new AtomicLong(0); + + AtomicBoolean callbackResult = new AtomicBoolean(false); + final Throwable[] throwable = new Throwable[1]; + + assertTrue(olf.isPresent()); + Futures.addCallback(olf.get(), new FutureCallback() { + @Override + public void onSuccess(Boolean result) { + numCallbackInvocationsSuccess.incrementAndGet(); + callbackResult.set(true); + } + + @Override + public void onFailure(Throwable t) { + throwable[0] = t; + numCallbackInvocationsFailure.incrementAndGet(); + callbackResult.set(true); + } + }); + + while (!callbackResult.get()) { + // Wait for the callback + Thread.sleep(DISK_CHECK_TIMEOUT); + } + + lock.unlock(); + + assertThat(numCallbackInvocationsFailure.get(), is(1L)); + assertThat(numCallbackInvocationsSuccess.get(), is(0L)); + assertTrue(throwable[0] instanceof TimeoutException); + } + + @Test (timeout = 2000) + public void testDiskCheckTimeoutInvokesOneCallbackOnly() throws Exception { + LOG.info("Executing {}", testName.getMethodName()); + + final DummyCheckable target = new DummyCheckable(); + final FakeTimer timer = new FakeTimer(); + ThrottledAsyncChecker checker = + new ThrottledAsyncChecker<>(timer, 0, DISK_CHECK_TIMEOUT, + getExecutorService()); + FutureCallback futureCallback = mock(FutureCallback.class); + + // Acquire lock to halt disk checker. Release after timeout occurs. + lock.lock(); + + final Optional> olf1 = checker + .schedule(target, true); + + assertTrue(olf1.isPresent()); + Futures.addCallback(olf1.get(), futureCallback); + + // Wait for the callback + Thread.sleep(DISK_CHECK_TIMEOUT); + + // Verify that timeout results in only 1 onFailure call and 0 onSuccess + // calls. + verify(futureCallback, times(1)).onFailure(any()); + verify(futureCallback, times(0)).onSuccess(any()); + + // Release lock so that target can acquire it. + lock.unlock(); + + final Optional> olf2 = checker + .schedule(target, true); + + assertTrue(olf2.isPresent()); + Futures.addCallback(olf2.get(), futureCallback); + + // Wait for the callback + Thread.sleep(DISK_CHECK_TIME); + + // Verify that normal check (dummy) results in only 1 onSuccess call. + // Number of times onFailure is invoked should remain the same - 1. + verify(futureCallback, times(1)).onFailure(any()); + verify(futureCallback, times(1)).onSuccess(any()); + } + + @Test (timeout = 1000) + public void testTimeoutExceptionIsNotThrownForGoodDisk() throws Exception { + LOG.info("Executing {}", testName.getMethodName()); + + final DummyCheckable target = new DummyCheckable(); + final FakeTimer timer = new FakeTimer(); + ThrottledAsyncChecker checker = + new ThrottledAsyncChecker<>(timer, 0, DISK_CHECK_TIMEOUT, + getExecutorService()); + + final Optional> olf = checker + .schedule(target, true); + + AtomicBoolean callbackResult = new AtomicBoolean(false); + final Throwable[] throwable = new Throwable[1]; + + assertTrue(olf.isPresent()); + Futures.addCallback(olf.get(), new FutureCallback() { + @Override + public void onSuccess(Boolean result) { + callbackResult.set(true); + } + + @Override + public void onFailure(Throwable t) { + throwable[0] = t; + callbackResult.set(true); + } + }); + + while (!callbackResult.get()) { + // Wait for the callback + Thread.sleep(DISK_CHECK_TIMEOUT); + } + + assertTrue(throwable[0] == null); + } + + /** + * A dummy Checkable that just returns true after acquiring lock. + */ + protected class DummyCheckable implements Checkable { + + @Override + public Boolean check(Boolean context) throws Exception { + // Wait to acquire lock + lock.lock(); + lock.unlock(); + return true; + } + } +} From 6d95866dcf278dd7789604787691fe8ee8d9cc9f Mon Sep 17 00:00:00 2001 From: Jian He Date: Thu, 16 Mar 2017 12:59:55 +0800 Subject: [PATCH 080/188] YARN-6332. Make RegistrySecurity use short user names for ZK ACLs. Contributed by Billie Rinaldi --- .../registry/client/impl/zk/RegistrySecurity.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java index bdb79be8a2a..23fadb57cba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java @@ -154,6 +154,8 @@ public class RegistrySecurity extends AbstractService { */ private final List systemACLs = new ArrayList(); + private boolean usesRealm = true; + /** * A list of digest ACLs which can be added to permissions * —and cleared later. @@ -232,6 +234,7 @@ public class RegistrySecurity extends AbstractService { // System Accounts String system = getOrFail(KEY_REGISTRY_SYSTEM_ACCOUNTS, DEFAULT_REGISTRY_SYSTEM_ACCOUNTS); + usesRealm = system.contains("@"); systemACLs.addAll(buildACLs(system, kerberosRealm, ZooDefs.Perms.ALL)); @@ -395,7 +398,12 @@ public class RegistrySecurity extends AbstractService { * @return a new ACL */ public ACL createSaslACL(UserGroupInformation ugi, int perms) { - String userName = ugi.getUserName(); + String userName = null; + if (usesRealm) { + userName = ugi.getUserName(); + } else { + userName = ugi.getShortUserName(); + } return new ACL(perms, new Id(SCHEME_SASL, userName)); } @@ -958,7 +966,7 @@ public class RegistrySecurity extends AbstractService { * @return an ACL for the user */ public ACL createACLfromUsername(String username, int perms) { - if (!username.contains("@")) { + if (usesRealm && !username.contains("@")) { username = username + "@" + kerberosRealm; if (LOG.isDebugEnabled()) { LOG.debug("Appending kerberos realm to make {}", username); From cd976b263be39bd4f75b1c94c09f82c862e04b30 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Thu, 16 Mar 2017 14:08:30 +0000 Subject: [PATCH 081/188] HDFS-11431. hadoop-hdfs-client JAR does not include ConfiguredFailoverProxyProvider. Contributed by Steven Rand. (cherry picked from commit b59dd4c79824280616641408b538e90f10eb49f8) --- hadoop-client-modules/hadoop-client/pom.xml | 26 ++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/hadoop-client-modules/hadoop-client/pom.xml b/hadoop-client-modules/hadoop-client/pom.xml index 6bc7232540a..e5dc0e41447 100644 --- a/hadoop-client-modules/hadoop-client/pom.xml +++ b/hadoop-client-modules/hadoop-client/pom.xml @@ -103,7 +103,7 @@ org.apache.hadoop - hadoop-hdfs-client + hadoop-hdfs compile @@ -126,6 +126,30 @@ javax.servlet javax.servlet-api + + io.netty + netty + + + io.netty + netty-all + + + xerces + xercesImpl + + + commons-daemon + commons-daemon + + + org.mortbay.jetty + jetty-util + + + org.fusesource.leveldbjni + leveldbjni-all + From 7114baddb627628a54cdab77f68504332a5a0e28 Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Thu, 16 Mar 2017 09:30:10 -0500 Subject: [PATCH 082/188] YARN-4051. ContainerKillEvent lost when container is still recovering and application finishes. Contributed by sandflee --- .../ContainerManagerImpl.java | 57 ++++++++++++++++--- .../application/ApplicationImpl.java | 4 +- .../containermanager/container/Container.java | 2 + .../container/ContainerImpl.java | 8 +++ .../TestContainerManagerRecovery.java | 18 +++--- .../nodemanager/webapp/MockContainer.java | 5 ++ 6 files changed, 76 insertions(+), 18 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java index 9f1655f1ef0..f3fe8ccc6a7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java @@ -402,8 +402,9 @@ public class ContainerManagerImpl extends CompositeService implements LOG.info("Recovering " + containerId + " in state " + rcs.getStatus() + " with exit code " + rcs.getExitCode()); - if (context.getApplications().containsKey(appId)) { - recoverActiveContainer(launchContext, token, rcs); + Application app = context.getApplications().get(appId); + if (app != null) { + recoverActiveContainer(app, launchContext, token, rcs); if (rcs.getRecoveryType() == RecoveredContainerType.KILL) { dispatcher.getEventHandler().handle( new ContainerKillEvent(containerId, ContainerExitStatus.ABORTED, @@ -423,7 +424,7 @@ public class ContainerManagerImpl extends CompositeService implements * Recover a running container. */ @SuppressWarnings("unchecked") - protected void recoverActiveContainer( + protected void recoverActiveContainer(Application app, ContainerLaunchContext launchContext, ContainerTokenIdentifier token, RecoveredContainerState rcs) throws IOException { Credentials credentials = YarnServerSecurityUtils.parseCredentials( @@ -431,8 +432,7 @@ public class ContainerManagerImpl extends CompositeService implements Container container = new ContainerImpl(getConfig(), dispatcher, launchContext, credentials, metrics, token, context, rcs); context.getContainers().put(token.getContainerID(), container); - dispatcher.getEventHandler().handle(new ApplicationContainerInitEvent( - container)); + app.handle(new ApplicationContainerInitEvent(container)); } private void waitForRecoveredContainers() throws InterruptedException { @@ -1286,6 +1286,10 @@ public class ContainerManagerImpl extends CompositeService implements + " is not handled by this NodeManager"); } } else { + if (container.isRecovering()) { + throw new NMNotYetReadyException("Container " + containerIDStr + + " is recovering, try later"); + } context.getNMStateStore().storeContainerKilled(containerID); container.sendKillEvent(ContainerExitStatus.KILLED_BY_APPMASTER, "Container killed by the ApplicationMaster."); @@ -1455,6 +1459,21 @@ public class ContainerManagerImpl extends CompositeService implements + " FINISH_APPS event"); continue; } + + boolean shouldDropEvent = false; + for (Container container : app.getContainers().values()) { + if (container.isRecovering()) { + LOG.info("drop FINISH_APPS event to " + appID + " because " + + "container " + container.getContainerId() + + " is recovering"); + shouldDropEvent = true; + break; + } + } + if (shouldDropEvent) { + continue; + } + String diagnostic = ""; if (appsFinishedEvent.getReason() == CMgrCompletedAppsEvent.Reason.ON_SHUTDOWN) { diagnostic = "Application killed on shutdown"; @@ -1469,10 +1488,32 @@ public class ContainerManagerImpl extends CompositeService implements case FINISH_CONTAINERS: CMgrCompletedContainersEvent containersFinishedEvent = (CMgrCompletedContainersEvent) event; - for (ContainerId container : containersFinishedEvent + for (ContainerId containerId : containersFinishedEvent .getContainersToCleanup()) { - this.dispatcher.getEventHandler().handle( - new ContainerKillEvent(container, + ApplicationId appId = + containerId.getApplicationAttemptId().getApplicationId(); + Application app = this.context.getApplications().get(appId); + if (app == null) { + LOG.warn("couldn't find app " + appId + " while processing" + + " FINISH_CONTAINERS event"); + continue; + } + + Container container = app.getContainers().get(containerId); + if (container == null) { + LOG.warn("couldn't find container " + containerId + + " while processing FINISH_CONTAINERS event"); + continue; + } + + if (container.isRecovering()) { + LOG.info("drop FINISH_CONTAINERS event to " + containerId + + " because container is recovering"); + continue; + } + + this.dispatcher.getEventHandler().handle( + new ContainerKillEvent(containerId, ContainerExitStatus.KILLED_BY_RESOURCEMANAGER, "Container Killed by ResourceManager")); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java index 112b43af878..444581c2bb2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java @@ -20,8 +20,8 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.application; import java.io.IOException; import java.util.EnumSet; -import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; @@ -89,7 +89,7 @@ public class ApplicationImpl implements Application { private LogAggregationContext logAggregationContext; Map containers = - new HashMap(); + new ConcurrentHashMap<>(); /** * The timestamp when the log aggregation has started for this application. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java index 8004f3393aa..bd3f06d1fcb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java @@ -92,4 +92,6 @@ public interface Container extends EventHandler { void sendLaunchEvent(); void sendKillEvent(int exitStatus, String description); + + boolean isRecovering(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java index cae30cde37d..055e12c57d6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java @@ -1756,4 +1756,12 @@ public class ContainerImpl implements Container { public void commitUpgrade() { this.reInitContext = null; } + + @Override + public boolean isRecovering() { + boolean isRecovering = ( + recoveredStatus != RecoveredContainerStatus.REQUESTED && + getContainerState() == ContainerState.NEW); + return isRecovering; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java index 31546f1c1e6..ff19e9ba317 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java @@ -86,6 +86,7 @@ import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationFinishEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationImpl; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; @@ -248,8 +249,8 @@ public class TestContainerManagerRecovery extends BaseContainerManagerTest { // simulate application completion List finishedApps = new ArrayList(); finishedApps.add(appId); - cm.handle(new CMgrCompletedAppsEvent(finishedApps, - CMgrCompletedAppsEvent.Reason.BY_RESOURCEMANAGER)); + app.handle(new ApplicationFinishEvent( + appId, "Application killed by ResourceManager")); waitForAppState(app, ApplicationState.APPLICATION_RESOURCES_CLEANINGUP); // restart and verify app is marked for finishing @@ -263,8 +264,8 @@ public class TestContainerManagerRecovery extends BaseContainerManagerTest { assertNotNull(app); // no longer saving FINISH_APP event in NM stateStore, // simulate by resending FINISH_APP event - cm.handle(new CMgrCompletedAppsEvent(finishedApps, - CMgrCompletedAppsEvent.Reason.BY_RESOURCEMANAGER)); + app.handle(new ApplicationFinishEvent( + appId, "Application killed by ResourceManager")); waitForAppState(app, ApplicationState.APPLICATION_RESOURCES_CLEANINGUP); assertTrue(context.getApplicationACLsManager().checkAccess( UserGroupInformation.createRemoteUser(modUser), @@ -335,8 +336,8 @@ public class TestContainerManagerRecovery extends BaseContainerManagerTest { // simulate application completion List finishedApps = new ArrayList(); finishedApps.add(appId); - cm.handle(new CMgrCompletedAppsEvent(finishedApps, - CMgrCompletedAppsEvent.Reason.BY_RESOURCEMANAGER)); + app.handle(new ApplicationFinishEvent( + appId, "Application killed by ResourceManager")); waitForAppState(app, ApplicationState.APPLICATION_RESOURCES_CLEANINGUP); app.handle(new ApplicationEvent(app.getAppId(), @@ -357,8 +358,9 @@ public class TestContainerManagerRecovery extends BaseContainerManagerTest { // no longer saving FINISH_APP event in NM stateStore, // simulate by resending FINISH_APP event - cm.handle(new CMgrCompletedAppsEvent(finishedApps, - CMgrCompletedAppsEvent.Reason.BY_RESOURCEMANAGER)); + app.handle(new ApplicationFinishEvent( + appId, "Application killed by ResourceManager")); + waitForAppState(app, ApplicationState.APPLICATION_RESOURCES_CLEANINGUP); // TODO need to figure out why additional APPLICATION_RESOURCES_CLEANEDUP // is needed. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java index 686a0d9fdd5..022baeac492 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java @@ -230,4 +230,9 @@ public class MockContainer implements Container { public void sendKillEvent(int exitStatus, String description) { } + + @Override + public boolean isRecovering() { + return false; + } } From ba62b50ebacd33b55eafc9db55a2fe5b4c80207a Mon Sep 17 00:00:00 2001 From: Kihwal Lee Date: Thu, 16 Mar 2017 10:53:38 -0500 Subject: [PATCH 083/188] HDFS-10601. Improve log message to include hostname when the NameNode is in safemode. Contributed by Kuhu Shukla. --- .../hdfs/server/namenode/FSNamesystem.java | 7 +++++- .../org/apache/hadoop/hdfs/TestSafeMode.java | 24 +++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 062896ce4f7..8f09740585a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -106,6 +106,7 @@ import java.io.PrintWriter; import java.io.StringWriter; import java.lang.management.ManagementFactory; import java.net.InetAddress; +import java.net.InetSocketAddress; import java.net.URI; import java.util.ArrayList; import java.util.Arrays; @@ -547,6 +548,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, */ private boolean manualSafeMode = false; private boolean resourceLowSafeMode = false; + private String nameNodeHostName = null; /** * Notify that loading of this FSDirectory is complete, and @@ -1116,6 +1118,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, dir.setINodeAttributeProvider(inodeAttributeProvider); } snapshotManager.registerMXBean(); + InetSocketAddress serviceAddress = NameNode.getServiceAddress(conf, true); + this.nameNodeHostName = (serviceAddress != null) ? + serviceAddress.getHostName() : ""; } /** @@ -1383,7 +1388,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, private SafeModeException newSafemodeException(String errorMsg) { return new SafeModeException(errorMsg + ". Name node is in safe " + - "mode.\n" + getSafeModeTip()); + "mode.\n" + getSafeModeTip() + " NamenodeHostName:" + nameNodeHostName); } boolean isPermissionEnabled() { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java index ada61b91a97..62dac74aea7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java @@ -304,6 +304,30 @@ public class TestSafeMode { } } + @Test + public void testSafeModeExceptionText() throws Exception { + final Path file1 = new Path("/file1"); + DFSTestUtil.createFile(fs, file1, 1024, (short)1, 0); + assertTrue("Could not enter SM", + dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER)); + try { + FSRun fsRun = new FSRun() { + @Override + public void run(FileSystem fileSystem) throws IOException { + ((DistributedFileSystem)fileSystem).setQuota(file1, 1, 1); + } + }; + fsRun.run(fs); + fail("Should not succeed with no exceptions!"); + } catch (RemoteException re) { + assertEquals(SafeModeException.class.getName(), re.getClassName()); + GenericTestUtils.assertExceptionContains( + NameNode.getServiceAddress(conf, true).getHostName(), re); + } catch (IOException ioe) { + fail("Encountered exception" + " " + StringUtils.stringifyException(ioe)); + } + } + /** * Run various fs operations while the NN is in safe mode, * assert that they are either allowed or fail as expected. From 79ede403eed49f77e3f0e4b103fc8619cac67168 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Thu, 16 Mar 2017 19:06:27 +0000 Subject: [PATCH 084/188] Revert "HDFS-11431. hadoop-hdfs-client JAR does not include ConfiguredFailoverProxyProvider." This reverts commit cd976b263be39bd4f75b1c94c09f82c862e04b30. --- hadoop-client-modules/hadoop-client/pom.xml | 26 +-------------------- 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/hadoop-client-modules/hadoop-client/pom.xml b/hadoop-client-modules/hadoop-client/pom.xml index e5dc0e41447..6bc7232540a 100644 --- a/hadoop-client-modules/hadoop-client/pom.xml +++ b/hadoop-client-modules/hadoop-client/pom.xml @@ -103,7 +103,7 @@ org.apache.hadoop - hadoop-hdfs + hadoop-hdfs-client compile @@ -126,30 +126,6 @@ javax.servlet javax.servlet-api - - io.netty - netty - - - io.netty - netty-all - - - xerces - xercesImpl - - - commons-daemon - commons-daemon - - - org.mortbay.jetty - jetty-util - - - org.fusesource.leveldbjni - leveldbjni-all - From 09ad8effb825eddbf0ee2ef591a0d16a58468f56 Mon Sep 17 00:00:00 2001 From: Jitendra Pandey Date: Thu, 16 Mar 2017 11:40:12 -0700 Subject: [PATCH 085/188] HDFS-11533. reuseAddress option should be used for child channels in Portmap and SimpleTcpServer. Contributed by Mukul Kumar Singh. --- .../src/main/java/org/apache/hadoop/oncrpc/SimpleTcpServer.java | 1 + .../src/main/java/org/apache/hadoop/portmap/Portmap.java | 1 + 2 files changed, 2 insertions(+) diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpServer.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpServer.java index f7ab52e3d50..bd48b158589 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpServer.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpServer.java @@ -81,6 +81,7 @@ public class SimpleTcpServer { }); server.setOption("child.tcpNoDelay", true); server.setOption("child.keepAlive", true); + server.setOption("child.reuseAddress", true); server.setOption("reuseAddress", true); // Listen to TCP port diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/Portmap.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/Portmap.java index 94d76d08fdd..7586fdad676 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/Portmap.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/Portmap.java @@ -110,6 +110,7 @@ final class Portmap { } }); tcpServer.setOption("reuseAddress", true); + tcpServer.setOption("child.reuseAddress", true); udpServer = new ConnectionlessBootstrap(new NioDatagramChannelFactory( Executors.newCachedThreadPool())); From 4812518b23cac496ab5cdad5258773bcd9728770 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Thu, 16 Mar 2017 15:07:38 -0700 Subject: [PATCH 086/188] HDFS-10530. BlockManager reconstruction work scheduling should correctly adhere to EC block placement policy. Contributed by Manoj Govindassamy and Rui Gao. --- .../server/blockmanagement/BlockManager.java | 2 +- .../hdfs/server/balancer/TestBalancer.java | 11 +- .../blockmanagement/TestBlockManager.java | 113 +++++++++++++++++- 3 files changed, 119 insertions(+), 7 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 5dc40fa10e7..be30e787cb2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -4179,7 +4179,7 @@ public class BlockManager implements BlockStatsMXBean { BlockPlacementPolicy placementPolicy = placementPolicies .getPolicy(blockType); int numReplicas = blockType == STRIPED ? ((BlockInfoStriped) storedBlock) - .getRealDataBlockNum() : storedBlock.getReplication(); + .getRealTotalBlockNum() : storedBlock.getReplication(); return placementPolicy.verifyBlockPlacement(locs, numReplicas) .isPlacementPolicySatisfied(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java index d2e8f4485e2..30a3a327d8c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java @@ -1918,7 +1918,7 @@ public class TestBalancer { } private void doTestBalancerWithStripedFile(Configuration conf) throws Exception { - int numOfDatanodes = dataBlocks + parityBlocks + 2; + int numOfDatanodes = dataBlocks + parityBlocks + 3; int numOfRacks = dataBlocks; long capacity = 20 * defaultBlockSize; long[] capacities = new long[numOfDatanodes]; @@ -1956,11 +1956,12 @@ public class TestBalancer { LocatedBlocks locatedBlocks = client.getBlockLocations(fileName, 0, fileLen); StripedFileTestUtil.verifyLocatedStripedBlocks(locatedBlocks, groupSize); - // add one datanode + // add datanodes in new rack String newRack = "/rack" + (++numOfRacks); - cluster.startDataNodes(conf, 1, true, null, - new String[]{newRack}, null, new long[]{capacity}); - totalCapacity += capacity; + cluster.startDataNodes(conf, 2, true, null, + new String[]{newRack, newRack}, null, + new long[]{capacity, capacity}); + totalCapacity += capacity*2; cluster.triggerHeartbeats(); // run balancer and validate results diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java index 00bea1c3882..36dafa5f78c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java @@ -40,6 +40,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.EnumSet; +import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map.Entry; @@ -55,6 +56,8 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSDataInputStream; @@ -68,6 +71,7 @@ import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; @@ -125,13 +129,14 @@ public class TestBlockManager { * of times trying to trigger the incorrect behavior. */ private static final int NUM_TEST_ITERS = 30; - private static final int BLOCK_SIZE = 64*1024; + private static final Log LOG = LogFactory.getLog(TestBlockManager.class); private FSNamesystem fsn; private BlockManager bm; private long mockINodeId; + @Before public void setupMockCluster() throws IOException { Configuration conf = new HdfsConfiguration(); @@ -1287,4 +1292,110 @@ public class TestBlockManager { isReplicaCorrupt(Mockito.any(BlockInfo.class), Mockito.any(DatanodeDescriptor.class)); } + + @Test (timeout = 300000) + public void testPlacementPolicySatisfied() throws Exception { + LOG.info("Starting testPlacementPolicySatisfied."); + final String[] initialRacks = new String[]{ + "/rack0", "/rack1", "/rack2", "/rack3", "/rack4", "/rack5"}; + final String[] initialHosts = new String[]{ + "host0", "host1", "host2", "host3", "host4", "host5"}; + final int numDataBlocks = StripedFileTestUtil.getDefaultECPolicy() + .getNumDataUnits(); + final int numParityBlocks = StripedFileTestUtil.getDefaultECPolicy() + .getNumParityUnits(); + final long blockSize = 64 * 1024; + Configuration conf = new Configuration(); + conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); + conf.setLong(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + StripedFileTestUtil.getDefaultECPolicy().getName()); + MiniDFSCluster cluster = null; + try { + cluster = new MiniDFSCluster.Builder(conf) + .racks(initialRacks) + .hosts(initialHosts) + .numDataNodes(initialRacks.length) + .build(); + cluster.waitActive(); + final DistributedFileSystem dfs = cluster.getFileSystem(); + final Path ecDir = new Path("/ec"); + final Path testFileUnsatisfied = new Path(ecDir, "test1"); + final Path testFileSatisfied = new Path(ecDir, "test2"); + cluster.getFileSystem().getClient().mkdirs(ecDir.toString(), null, true); + cluster.getFileSystem().getClient() + .setErasureCodingPolicy(ecDir.toString(), + StripedFileTestUtil.getDefaultECPolicy().getName()); + long fileLen = blockSize * numDataBlocks; + + // Create a file to be stored in 6 racks. + DFSTestUtil.createFile(dfs, testFileUnsatisfied, fileLen, (short) 1, 1); + // Block placement policy should be satisfied as rack count + // is less than numDataBlocks + numParityBlocks. + verifyPlacementPolicy(cluster, testFileUnsatisfied, true); + + LOG.info("Adding 3 new hosts in the existing racks."); + cluster.startDataNodes(conf, 3, true, null, + new String[]{"/rack3", "/rack4", "/rack5"}, + new String[]{"host3-2", "host4-2", "host5-2"}, null); + cluster.triggerHeartbeats(); + + LOG.info("Waiting for EC reconstruction to complete."); + DFSTestUtil.waitForReplication(dfs, testFileUnsatisfied, + (short)(numDataBlocks + numParityBlocks), 30 * 1000); + // Block placement policy should still be satisfied + // as there are only 6 racks. + verifyPlacementPolicy(cluster, testFileUnsatisfied, true); + + LOG.info("Adding 3 new hosts in 3 new racks."); + cluster.startDataNodes(conf, 3, true, null, + new String[]{"/rack6", "/rack7", "/rack8"}, + new String[]{"host6", "host7", "host8"}, + null); + cluster.triggerHeartbeats(); + // Addition of new racks can make the existing EC files block + // placements unsatisfied and there is NO automatic block + // reconstruction for this yet. + // TODO: + // Verify for block placement satisfied once the automatic + // block reconstruction is implemented. + verifyPlacementPolicy(cluster, testFileUnsatisfied, false); + + // Create a new file + DFSTestUtil.createFile(dfs, testFileSatisfied, fileLen, (short) 1, 1); + // The new file should be rightly placed on all 9 racks + // and the block placement policy should be satisfied. + verifyPlacementPolicy(cluster, testFileUnsatisfied, false); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } + + private void verifyPlacementPolicy(final MiniDFSCluster cluster, + final Path file, boolean isBlockPlacementSatisfied) throws IOException { + DistributedFileSystem dfs = cluster.getFileSystem(); + BlockManager blockManager = cluster.getNamesystem().getBlockManager(); + LocatedBlock lb = DFSTestUtil.getAllBlocks(dfs, file).get(0); + BlockInfo blockInfo = + blockManager.getStoredBlock(lb.getBlock().getLocalBlock()); + Iterator itr = blockInfo.getStorageInfos(); + LOG.info("Block " + blockInfo + " storages: "); + while (itr.hasNext()) { + DatanodeStorageInfo dn = itr.next(); + LOG.info(" Rack: " + dn.getDatanodeDescriptor().getNetworkLocation() + + ", DataNode: " + dn.getDatanodeDescriptor().getXferAddr()); + } + if (isBlockPlacementSatisfied) { + assertTrue("Block group of " + file + "should be placement" + + " policy satisfied, currently!", + blockManager.isPlacementPolicySatisfied(blockInfo)); + } else { + assertFalse("Block group of " + file + " should be placement" + + " policy unsatisfied, currently!", + blockManager.isPlacementPolicySatisfied(blockInfo)); + } + } + } From c04fb35c4542a455813f53ce966f9b860ac41df1 Mon Sep 17 00:00:00 2001 From: Mingliang Liu Date: Thu, 16 Mar 2017 16:08:32 -0700 Subject: [PATCH 087/188] HADOOP-14191. Duplicate hadoop-minikdc dependency in hadoop-common module. Contributed by Xiaobing Zhou --- hadoop-common-project/hadoop-common/pom.xml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 0db68412e27..961c9028393 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -45,11 +45,6 @@ hadoop-annotations compile - - org.apache.hadoop - hadoop-minikdc - test - com.google.guava guava From 75368150395901f65a4698e84be4e7bbdcba94fa Mon Sep 17 00:00:00 2001 From: Xiaobing Zhou Date: Thu, 16 Mar 2017 18:13:43 -0700 Subject: [PATCH 088/188] HDFS-10394. move declaration of okhttp version from hdfs-client to hadoop-project POM. Contributed by Xiaobing Zhou. --- hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 1 - hadoop-project/pom.xml | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml index f8aa3716880..9fd7bb4dcee 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml @@ -37,7 +37,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> com.squareup.okhttp okhttp - 2.4.0 org.apache.hadoop diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 572f0e6eb29..fbe961a075b 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -134,6 +134,11 @@ + + com.squareup.okhttp + okhttp + 2.4.0 + jdiff jdiff From 86035c16447eb0ad7989975639f83bc4b4b26a27 Mon Sep 17 00:00:00 2001 From: Mingliang Liu Date: Thu, 16 Mar 2017 15:24:46 -0700 Subject: [PATCH 089/188] HADOOP-14192. Aliyun OSS FileSystem contract test should implement getTestBaseDir(). Contributed by Mingliang Liu --- .../oss/TestAliyunOSSFileSystemContract.java | 35 +++---------------- 1 file changed, 4 insertions(+), 31 deletions(-) diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java index 3ebf5077b79..69ce694ab89 100644 --- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java +++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java @@ -38,8 +38,8 @@ import java.io.IOException; public class TestAliyunOSSFileSystemContract extends FileSystemContractBaseTest { public static final String TEST_FS_OSS_NAME = "test.fs.oss.name"; - private static String testRootPath = - AliyunOSSTestUtils.generateUniqueTestPath(); + private static Path testRootPath = + new Path(AliyunOSSTestUtils.generateUniqueTestPath()); @Override public void setUp() throws Exception { @@ -49,20 +49,8 @@ public class TestAliyunOSSFileSystemContract } @Override - public void tearDown() throws Exception { - if (fs != null) { - fs.delete(super.path(testRootPath), true); - } - super.tearDown(); - } - - @Override - protected Path path(String path) { - if (path.startsWith("/")) { - return super.path(testRootPath + path); - } else { - return super.path(testRootPath + "/" + path); - } + public Path getTestBaseDir() { + return testRootPath; } @Override @@ -228,19 +216,4 @@ public class TestAliyunOSSFileSystemContract } } - public void testWorkingDirectory() throws Exception { - Path workDir = super.path(this.getDefaultWorkingDirectory()); - assertEquals(workDir, this.fs.getWorkingDirectory()); - this.fs.setWorkingDirectory(super.path(".")); - assertEquals(workDir, this.fs.getWorkingDirectory()); - this.fs.setWorkingDirectory(super.path("..")); - assertEquals(workDir.getParent(), this.fs.getWorkingDirectory()); - Path relativeDir = super.path("hadoop"); - this.fs.setWorkingDirectory(relativeDir); - assertEquals(relativeDir, this.fs.getWorkingDirectory()); - Path absoluteDir = super.path("/test/hadoop"); - this.fs.setWorkingDirectory(absoluteDir); - assertEquals(absoluteDir, this.fs.getWorkingDirectory()); - } - } From 92ea6d74ecd84b86771e285902dd2a80bccbe3ea Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Fri, 17 Mar 2017 11:26:02 -0700 Subject: [PATCH 090/188] HDFS-11516. Admin command line should print message to stderr in failure case. Contributed by Kai Sasaki. --- .../org/apache/hadoop/cli/CLITestHelper.java | 17 +++++++++- .../hadoop/cli/util/CommandExecutor.java | 4 +-- .../apache/hadoop/cli/util/FSCmdExecutor.java | 4 +-- .../apache/hadoop/hdfs/tools/AdminHelper.java | 6 ++-- .../apache/hadoop/hdfs/tools/CryptoAdmin.java | 2 +- .../cli/util/CacheAdminCmdExecutor.java | 4 +-- .../cli/util/CryptoAdminCmdExecutor.java | 4 +-- .../cli/util/ErasureCodingCliCmdExecutor.java | 4 +-- .../src/test/resources/testCryptoConf.xml | 31 +++++++++++++++++++ .../test/resources/testErasureCodingConf.xml | 16 ++++++++++ 10 files changed, 77 insertions(+), 15 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/CLITestHelper.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/CLITestHelper.java index 89d4e30ce26..ada4cd80e48 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/CLITestHelper.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/CLITestHelper.java @@ -298,6 +298,11 @@ public class CLITestHelper { return compareOutput; } + + private boolean compareTextExitCode(ComparatorData compdata, + Result cmdResult) { + return compdata.getExitCode() == cmdResult.getExitCode(); + } /*********************************** ************* TESTS RUNNER @@ -330,10 +335,17 @@ public class CLITestHelper { final String comptype = cd.getComparatorType(); boolean compareOutput = false; + boolean compareExitCode = false; if (! comptype.equalsIgnoreCase("none")) { compareOutput = compareTestOutput(cd, cmdResult); - overallTCResult &= compareOutput; + if (cd.getExitCode() == -1) { + // No need to check exit code if not specified + compareExitCode = true; + } else { + compareExitCode = compareTextExitCode(cd, cmdResult); + } + overallTCResult &= (compareOutput & compareExitCode); } cd.setExitCode(cmdResult.getExitCode()); @@ -391,6 +403,7 @@ public class CLITestHelper { testComparators = new ArrayList(); } else if (qName.equals("comparator")) { comparatorData = new ComparatorData(); + comparatorData.setExitCode(-1); } charString = ""; } @@ -422,6 +435,8 @@ public class CLITestHelper { comparatorData.setComparatorType(charString); } else if (qName.equals("expected-output")) { comparatorData.setExpectedOutput(charString); + } else if (qName.equals("expected-exit-code")) { + comparatorData.setExitCode(Integer.valueOf(charString)); } else if (qName.equals("test")) { if (!Shell.WINDOWS || runOnWindows) { testsFromConfigFile.add(td); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CommandExecutor.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CommandExecutor.java index 79df284045b..5ef129cdc87 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CommandExecutor.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CommandExecutor.java @@ -75,7 +75,7 @@ public abstract class CommandExecutor { System.setErr(new PrintStream(bao)); try { - execute(cmd); + exitCode = execute(cmd); } catch (Exception e) { e.printStackTrace(); lastException = e; @@ -87,7 +87,7 @@ public abstract class CommandExecutor { return new Result(bao.toString(), exitCode, lastException, cmd); } - protected abstract void execute(final String cmd) throws Exception; + protected abstract int execute(String cmd) throws Exception; public static class Result { final String commandOutput; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/FSCmdExecutor.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/FSCmdExecutor.java index 98237ac7263..e2693bb6eaf 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/FSCmdExecutor.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/FSCmdExecutor.java @@ -30,8 +30,8 @@ public class FSCmdExecutor extends CommandExecutor { } @Override - protected void execute(final String cmd) throws Exception{ + protected int execute(final String cmd) throws Exception{ String[] args = getCommandAsArgs(cmd, "NAMENODE", this.namenode); - ToolRunner.run(shell, args); + return ToolRunner.run(shell, args); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/AdminHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/AdminHelper.java index 8bab5505ef5..2c5fb0e1e82 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/AdminHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/AdminHelper.java @@ -176,11 +176,11 @@ public class AdminHelper { for (AdminHelper.Command command : commands) { System.err.println(command.getLongUsage()); } - return 0; + return 1; } if (args.size() != 1) { - System.out.println("You must give exactly one argument to -help."); - return 0; + System.err.println("You must give exactly one argument to -help."); + return 1; } final String commandName = args.get(0); // prepend a dash to match against the command names diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java index 4c7335ffb27..14abf6e074f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java @@ -233,7 +233,7 @@ public class CryptoAdmin extends Configured implements Tool { final FileEncryptionInfo fei = admin.getFileEncryptionInfo(p); if (fei == null) { - System.out.println("No FileEncryptionInfo found for path " + path); + System.err.println("No FileEncryptionInfo found for path " + path); return 2; } System.out.println(fei.toStringStable()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/CacheAdminCmdExecutor.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/CacheAdminCmdExecutor.java index 922020faf84..75efb3edf4c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/CacheAdminCmdExecutor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/CacheAdminCmdExecutor.java @@ -30,8 +30,8 @@ public class CacheAdminCmdExecutor extends CommandExecutor { } @Override - protected void execute(final String cmd) throws Exception { + protected int execute(final String cmd) throws Exception { String[] args = getCommandAsArgs(cmd, "NAMENODE", this.namenode); - ToolRunner.run(admin, args); + return ToolRunner.run(admin, args); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/CryptoAdminCmdExecutor.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/CryptoAdminCmdExecutor.java index f781bf8d877..0ab5b0ea282 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/CryptoAdminCmdExecutor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/CryptoAdminCmdExecutor.java @@ -30,8 +30,8 @@ public class CryptoAdminCmdExecutor extends CommandExecutor { } @Override - protected void execute(final String cmd) throws Exception { + protected int execute(final String cmd) throws Exception { String[] args = getCommandAsArgs(cmd, "NAMENODE", this.namenode); - ToolRunner.run(admin, args); + return ToolRunner.run(admin, args); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/ErasureCodingCliCmdExecutor.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/ErasureCodingCliCmdExecutor.java index 59b2a73e2d3..e39b4c9cbdd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/ErasureCodingCliCmdExecutor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/ErasureCodingCliCmdExecutor.java @@ -30,8 +30,8 @@ public class ErasureCodingCliCmdExecutor extends CommandExecutor { } @Override - protected void execute(final String cmd) throws Exception { + protected int execute(final String cmd) throws Exception { String[] args = getCommandAsArgs(cmd, "NAMENODE", this.namenode); - ToolRunner.run(admin, args); + return ToolRunner.run(admin, args); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml index 7e372becaf8..f9bb29eb9e4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml @@ -46,6 +46,37 @@ + + Test help usage + + -help + + + + + + SubstringComparator + [-createZone -keyName + + + + + + Test extra help argument + + -help arg1 arg2 + + + + + + SubstringComparator + You must give exactly one argument to -help. + 1 + + + + Test create ez, dir doesn't exist diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml index 8dda6c1af22..0a71109d789 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml @@ -71,6 +71,22 @@ + + help: help with extra argument + + -help arg1 arg2 + + + + + + SubstringComparator + You must give exactly one argument to -help. + 1 + + + + help: setPolicy command From 4a8e3045027036afebbcb80f23b7a2886e56c255 Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Fri, 17 Mar 2017 14:06:35 -0500 Subject: [PATCH 091/188] YARN-6217. TestLocalCacheDirectoryManager test timeout is too aggressive. Contributed by Miklos Szegedi --- .../localizer/TestLocalCacheDirectoryManager.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalCacheDirectoryManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalCacheDirectoryManager.java index df00f9e7ed0..82d9a946661 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalCacheDirectoryManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalCacheDirectoryManager.java @@ -33,7 +33,7 @@ import org.junit.Test; public class TestLocalCacheDirectoryManager { - @Test(timeout = 10000) + @Test public void testHierarchicalSubDirectoryCreation() { // setting per directory file limit to 1. YarnConfiguration conf = new YarnConfiguration(); @@ -73,7 +73,7 @@ public class TestLocalCacheDirectoryManager { Assert.assertEquals(testPath2, hDir.getRelativePathForLocalization()); } - @Test(timeout = 10000) + @Test public void testMinimumPerDirectoryFileLimit() { YarnConfiguration conf = new YarnConfiguration(); conf.set(YarnConfiguration.NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY, "1"); @@ -98,7 +98,7 @@ public class TestLocalCacheDirectoryManager { } - @Test(timeout = 1000) + @Test public void testDirectoryStateChangeFromFullToNonFull() { YarnConfiguration conf = new YarnConfiguration(); conf.set(YarnConfiguration.NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY, "40"); From 7f8e928400db954cfe37a3bd35762e1b310dcb8e Mon Sep 17 00:00:00 2001 From: Hanisha Koneru Date: Fri, 17 Mar 2017 15:42:25 -0700 Subject: [PATCH 092/188] HDFS-11517. Expose slow disks via DataNode JMX. Contributed by Hanisha Koneru --- .../hadoop/hdfs/server/datanode/DataNode.java | 14 ++++++++ .../hdfs/server/datanode/DataNodeMXBean.java | 7 ++++ .../datanode/metrics/DataNodeDiskMetrics.java | 13 +++++++- .../server/datanode/TestDataNodeMXBean.java | 33 ++++++++++++++++++- 4 files changed, 65 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 5a82850c3f1..4b7e052b1d1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -1821,6 +1821,10 @@ public class DataNode extends ReconfigurableBase public DataNodeMetrics getMetrics() { return metrics; } + + public DataNodeDiskMetrics getDiskMetrics() { + return diskMetrics; + } public DataNodePeerMetrics getPeerMetrics() { return peerMetrics; @@ -3520,4 +3524,14 @@ public class DataNode extends ReconfigurableBase return peerMetrics != null ? peerMetrics.dumpSendPacketDownstreamAvgInfoAsJson() : null; } + + @Override // DataNodeMXBean + public String getSlowDisks() { + if (diskMetrics == null) { + //Disk Stats not enabled + return null; + } + Set slowDisks = diskMetrics.getDiskOutliersStats().keySet(); + return JSON.toString(slowDisks); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeMXBean.java index fb79a86d287..c86fe449131 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeMXBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeMXBean.java @@ -132,4 +132,11 @@ public interface DataNodeMXBean { *

    */ String getSendPacketDownstreamAvgInfo(); + + /** + * Gets the slow disks in the Datanode. + * + * @return list of slow disks + */ + String getSlowDisks(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java index 85e2bd9f23c..20a356790bf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hdfs.server.datanode.metrics; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import org.apache.hadoop.classification.InterfaceAudience; @@ -52,7 +54,8 @@ public class DataNodeDiskMetrics { private volatile boolean shouldRun; private OutlierDetector slowDiskDetector; private Daemon slowDiskDetectionDaemon; - private volatile Map> diskOutliersStats; + private volatile Map> + diskOutliersStats = Maps.newHashMap(); public DataNodeDiskMetrics(DataNode dn, long diskOutlierDetectionIntervalMs) { this.dn = dn; @@ -178,4 +181,12 @@ public class DataNodeDiskMetrics { LOG.error("Disk Outlier Detection daemon did not shutdown", e); } } + + /** + * Use only for testing. + */ + @VisibleForTesting + public void addSlowDiskForTesting(String slowDiskPath) { + diskOutliersStats.put(slowDiskPath, ImmutableMap.of()); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMXBean.java index 3474290c062..b6277d60a8f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMXBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMXBean.java @@ -35,6 +35,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.test.GenericTestUtils; @@ -104,8 +105,12 @@ public class TestDataNodeMXBean { String bpActorInfo = (String)mbs.getAttribute(mxbeanName, "BPServiceActorInfo"); Assert.assertEquals(datanode.getBPServiceActorInfo(), bpActorInfo); + String slowDisks = (String)mbs.getAttribute(mxbeanName, "SlowDisks"); + Assert.assertEquals(datanode.getSlowDisks(), slowDisks); } finally { - if (cluster != null) {cluster.shutdown();} + if (cluster != null) { + cluster.shutdown(); + } } } @@ -209,4 +214,30 @@ public class TestDataNodeMXBean { } return totalBlocks; } + + @Test + public void testDataNodeMXBeanSlowDisksEnabled() throws Exception { + Configuration conf = new Configuration(); + conf.setDouble(DFSConfigKeys + .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY, 1.0); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); + + try { + List datanodes = cluster.getDataNodes(); + Assert.assertEquals(datanodes.size(), 1); + DataNode datanode = datanodes.get(0); + String slowDiskPath = "test/data1/slowVolume"; + datanode.getDiskMetrics().addSlowDiskForTesting(slowDiskPath); + + MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); + ObjectName mxbeanName = new ObjectName( + "Hadoop:service=DataNode,name=DataNodeInfo"); + + String slowDisks = (String)mbs.getAttribute(mxbeanName, "SlowDisks"); + Assert.assertEquals(datanode.getSlowDisks(), slowDisks); + Assert.assertTrue(slowDisks.contains(slowDiskPath)); + } finally { + if (cluster != null) {cluster.shutdown();} + } + } } From e1a99802fc5c2f2ad3ceb000378db1c0133069b3 Mon Sep 17 00:00:00 2001 From: Erik Krogen Date: Fri, 17 Mar 2017 17:11:02 -0700 Subject: [PATCH 093/188] HADOOP-14169. Implement listStatusIterator, listLocatedStatus for ViewFs. Contributed by Erik Krogen. --- .../apache/hadoop/fs/viewfs/ChRootedFs.java | 14 +++ .../org/apache/hadoop/fs/viewfs/ViewFs.java | 77 ++++++++++++---- .../hadoop/fs/viewfs/ViewFsBaseTest.java | 92 +++++++++++++++++-- 3 files changed, 156 insertions(+), 27 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java index f07e07f199d..b7e47c21f13 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java @@ -37,8 +37,10 @@ import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FsServerDefaults; import org.apache.hadoop.fs.FsStatus; +import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Options.ChecksumOpt; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.UnresolvedLinkException; import org.apache.hadoop.fs.XAttrSetFlag; import org.apache.hadoop.fs.permission.AclEntry; @@ -236,6 +238,18 @@ class ChRootedFs extends AbstractFileSystem { return myFs.listStatus(fullPath(f)); } + @Override + public RemoteIterator listStatusIterator(final Path f) + throws IOException, UnresolvedLinkException { + return myFs.listStatusIterator(fullPath(f)); + } + + @Override + public RemoteIterator listLocatedStatus(final Path f) + throws IOException, UnresolvedLinkException { + return myFs.listLocatedStatus(fullPath(f)); + } + @Override public void mkdir(final Path dir, final FsPermission permission, final boolean createParent) throws IOException, UnresolvedLinkException { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java index d5f8e81f26d..1a54a81a0f1 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java @@ -44,6 +44,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FsConstants; import org.apache.hadoop.fs.FsServerDefaults; import org.apache.hadoop.fs.FsStatus; +import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Options.ChecksumOpt; import org.apache.hadoop.fs.ParentNotDirectoryException; import org.apache.hadoop.fs.Path; @@ -388,26 +389,32 @@ public class ViewFs extends AbstractFileSystem { if (res.isInternalDir()) { return fsIter; } - - return new RemoteIterator() { - final RemoteIterator myIter; - final ChRootedFs targetFs; - { // Init - myIter = fsIter; - targetFs = (ChRootedFs) res.targetFileSystem; - } - + + return new WrappingRemoteIterator(res, fsIter, f) { @Override - public boolean hasNext() throws IOException { - return myIter.hasNext(); + public FileStatus getViewFsFileStatus(FileStatus stat, Path newPath) { + return new ViewFsFileStatus(stat, newPath); } - + }; + } + + @Override + public RemoteIterator listLocatedStatus(final Path f) + throws AccessControlException, FileNotFoundException, + UnresolvedLinkException, IOException { + final InodeTree.ResolveResult res = + fsState.resolve(getUriPath(f), true); + final RemoteIterator fsIter = + res.targetFileSystem.listLocatedStatus(res.remainingPath); + if (res.isInternalDir()) { + return fsIter; + } + + return new WrappingRemoteIterator(res, fsIter, f) { @Override - public FileStatus next() throws IOException { - FileStatus status = myIter.next(); - String suffix = targetFs.stripOutRoot(status.getPath()); - return new ViewFsFileStatus(status, makeQualified( - suffix.length() == 0 ? f : new Path(res.resolvedPath, suffix))); + public LocatedFileStatus getViewFsFileStatus(LocatedFileStatus stat, + Path newPath) { + return new ViewFsLocatedFileStatus(stat, newPath); } }; } @@ -773,6 +780,42 @@ public class ViewFs extends AbstractFileSystem { return res.targetFileSystem.getStoragePolicy(res.remainingPath); } + /** + * Helper class to perform some transformation on results returned + * from a RemoteIterator. + */ + private abstract class WrappingRemoteIterator + implements RemoteIterator { + private final String resolvedPath; + private final ChRootedFs targetFs; + private final RemoteIterator innerIter; + private final Path originalPath; + + WrappingRemoteIterator(InodeTree.ResolveResult res, + RemoteIterator innerIter, Path originalPath) { + this.resolvedPath = res.resolvedPath; + this.targetFs = (ChRootedFs)res.targetFileSystem; + this.innerIter = innerIter; + this.originalPath = originalPath; + } + + @Override + public boolean hasNext() throws IOException { + return innerIter.hasNext(); + } + + @Override + public T next() throws IOException { + T status = innerIter.next(); + String suffix = targetFs.stripOutRoot(status.getPath()); + Path newPath = makeQualified(suffix.length() == 0 ? originalPath + : new Path(resolvedPath, suffix)); + return getViewFsFileStatus(status, newPath); + } + + protected abstract T getViewFsFileStatus(T status, Path newPath); + } + /* * An instance of this class represents an internal dir of the viewFs * ie internal dir of the mount table. diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java index 16d26442d14..8f6df32e777 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java @@ -25,6 +25,13 @@ import static org.apache.hadoop.fs.FileContextTestHelper.isFile; import static org.apache.hadoop.fs.viewfs.Constants.PERMISSION_555; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyBoolean; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import java.io.FileNotFoundException; import java.io.IOException; @@ -32,13 +39,16 @@ import java.net.URI; import java.net.URISyntaxException; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.AbstractFileSystem; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileContextTestHelper; +import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.FileContextTestHelper.fileType; import org.apache.hadoop.fs.FileStatus; @@ -56,7 +66,6 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.mockito.Mockito; /** @@ -319,6 +328,16 @@ abstract public class ViewFsBaseTest { } } Assert.assertTrue(dirFooPresent); + RemoteIterator dirLocatedContents = + fcView.listLocatedStatus(new Path("/targetRoot/")); + dirFooPresent = false; + while (dirLocatedContents.hasNext()) { + FileStatus fileStatus = dirLocatedContents.next(); + if (fileStatus.getPath().getName().equals("dirFoo")) { + dirFooPresent = true; + } + } + Assert.assertTrue(dirFooPresent); } // rename across mount points that point to same target also fail @@ -450,24 +469,23 @@ abstract public class ViewFsBaseTest { } @Test - public void testGetFileChecksum() throws AccessControlException - , UnresolvedLinkException, IOException { - AbstractFileSystem mockAFS = Mockito.mock(AbstractFileSystem.class); + public void testGetFileChecksum() throws AccessControlException, + UnresolvedLinkException, IOException { + AbstractFileSystem mockAFS = mock(AbstractFileSystem.class); InodeTree.ResolveResult res = new InodeTree.ResolveResult(null, mockAFS , null, new Path("someFile")); @SuppressWarnings("unchecked") - InodeTree fsState = Mockito.mock(InodeTree.class); - Mockito.when(fsState.resolve(Mockito.anyString() - , Mockito.anyBoolean())).thenReturn(res); - ViewFs vfs = Mockito.mock(ViewFs.class); + InodeTree fsState = mock(InodeTree.class); + when(fsState.resolve(anyString(), anyBoolean())).thenReturn(res); + ViewFs vfs = mock(ViewFs.class); vfs.fsState = fsState; - Mockito.when(vfs.getFileChecksum(new Path("/tmp/someFile"))) + when(vfs.getFileChecksum(new Path("/tmp/someFile"))) .thenCallRealMethod(); vfs.getFileChecksum(new Path("/tmp/someFile")); - Mockito.verify(mockAFS).getFileChecksum(new Path("someFile")); + verify(mockAFS).getFileChecksum(new Path("someFile")); } @Test(expected=FileNotFoundException.class) @@ -820,4 +838,58 @@ abstract public class ViewFsBaseTest { } }); } + + // Confirm that listLocatedStatus is delegated properly to the underlying + // AbstractFileSystem to allow for optimizations + @Test + public void testListLocatedStatus() throws IOException { + final Path mockTarget = new Path("mockfs://listLocatedStatus/foo"); + final Path mountPoint = new Path("/fooMount"); + final Configuration newConf = new Configuration(); + newConf.setClass("fs.AbstractFileSystem.mockfs.impl", MockFs.class, + AbstractFileSystem.class); + ConfigUtil.addLink(newConf, mountPoint.toString(), mockTarget.toUri()); + FileContext.getFileContext(URI.create("viewfs:///"), newConf) + .listLocatedStatus(mountPoint); + AbstractFileSystem mockFs = MockFs.getMockFs(mockTarget.toUri()); + verify(mockFs).listLocatedStatus(new Path(mockTarget.toUri().getPath())); + verify(mockFs, never()).listStatus(any(Path.class)); + verify(mockFs, never()).listStatusIterator(any(Path.class)); + } + + // Confirm that listStatus is delegated properly to the underlying + // AbstractFileSystem's listStatusIterator to allow for optimizations + @Test + public void testListStatusIterator() throws IOException { + final Path mockTarget = new Path("mockfs://listStatusIterator/foo"); + final Path mountPoint = new Path("/fooMount"); + final Configuration newConf = new Configuration(); + newConf.setClass("fs.AbstractFileSystem.mockfs.impl", MockFs.class, + AbstractFileSystem.class); + ConfigUtil.addLink(newConf, mountPoint.toString(), mockTarget.toUri()); + FileContext.getFileContext(URI.create("viewfs:///"), newConf) + .listStatus(mountPoint); + AbstractFileSystem mockFs = MockFs.getMockFs(mockTarget.toUri()); + verify(mockFs).listStatusIterator(new Path(mockTarget.toUri().getPath())); + verify(mockFs, never()).listStatus(any(Path.class)); + } + + static class MockFs extends ChRootedFs { + private static Map fsCache = new HashMap<>(); + MockFs(URI uri, Configuration conf) throws URISyntaxException { + super(getMockFs(uri), new Path("/")); + } + static AbstractFileSystem getMockFs(URI uri) { + AbstractFileSystem mockFs = fsCache.get(uri.getAuthority()); + if (mockFs == null) { + mockFs = mock(AbstractFileSystem.class); + when(mockFs.getUri()).thenReturn(uri); + when(mockFs.getUriDefaultPort()).thenReturn(1); + when(mockFs.getUriPath(any(Path.class))).thenCallRealMethod(); + when(mockFs.isValidName(anyString())).thenReturn(true); + fsCache.put(uri.getAuthority(), mockFs); + } + return mockFs; + } + } } From ffa160ddb824cbcb8ab6b10ee1414507686e3c63 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Fri, 17 Mar 2017 19:02:24 -0700 Subject: [PATCH 094/188] HDFS-11547. Add logs for slow BlockReceiver while writing data to disk. Contributed by Xiaobing Zhou. --- .../org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java index 5852403ac2d..bb6bd5570e2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java @@ -711,6 +711,10 @@ class BlockReceiver implements Closeable { streams.writeDataToDisk(dataBuf.array(), startByteToDisk, numBytesToDisk); long duration = Time.monotonicNow() - begin; + if (duration > datanodeSlowLogThresholdMs) { + LOG.warn("Slow BlockReceiver write data to disk cost:" + duration + + "ms (threshold=" + datanodeSlowLogThresholdMs + "ms)"); + } if (duration > maxWriteToDiskMs) { maxWriteToDiskMs = duration; From 34a931c0fa5853fbcaadf0211d6bbee881715748 Mon Sep 17 00:00:00 2001 From: Naganarasimha Date: Mon, 20 Mar 2017 00:04:23 +0530 Subject: [PATCH 095/188] MAPREDUCE-6865. Fix typo in javadoc for DistributedCache. Contributed by Attila Sasvari. --- .../org/apache/hadoop/filecache/DistributedCache.java | 8 ++++---- .../hadoop/mapreduce/filecache/DistributedCache.java | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/filecache/DistributedCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/filecache/DistributedCache.java index 65d43292f93..90f864917e1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/filecache/DistributedCache.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/filecache/DistributedCache.java @@ -83,11 +83,11 @@ import org.apache.hadoop.mapreduce.MRJobConfig; * JobConf job = new JobConf(); * DistributedCache.addCacheFile(new URI("/myapp/lookup.dat#lookup.dat"), * job); - * DistributedCache.addCacheArchive(new URI("/myapp/map.zip", job); + * DistributedCache.addCacheArchive(new URI("/myapp/map.zip"), job); * DistributedCache.addFileToClassPath(new Path("/myapp/mylib.jar"), job); - * DistributedCache.addCacheArchive(new URI("/myapp/mytar.tar", job); - * DistributedCache.addCacheArchive(new URI("/myapp/mytgz.tgz", job); - * DistributedCache.addCacheArchive(new URI("/myapp/mytargz.tar.gz", job); + * DistributedCache.addCacheArchive(new URI("/myapp/mytar.tar"), job); + * DistributedCache.addCacheArchive(new URI("/myapp/mytgz.tgz"), job); + * DistributedCache.addCacheArchive(new URI("/myapp/mytargz.tar.gz"), job); * * 3. Use the cached files in the {@link org.apache.hadoop.mapred.Mapper} * or {@link org.apache.hadoop.mapred.Reducer}: diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java index 0c43633c1f9..532e3ad3d7c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java @@ -85,11 +85,11 @@ import java.net.URI; * JobConf job = new JobConf(); * DistributedCache.addCacheFile(new URI("/myapp/lookup.dat#lookup.dat"), * job); - * DistributedCache.addCacheArchive(new URI("/myapp/map.zip", job); + * DistributedCache.addCacheArchive(new URI("/myapp/map.zip"), job); * DistributedCache.addFileToClassPath(new Path("/myapp/mylib.jar"), job); - * DistributedCache.addCacheArchive(new URI("/myapp/mytar.tar", job); - * DistributedCache.addCacheArchive(new URI("/myapp/mytgz.tgz", job); - * DistributedCache.addCacheArchive(new URI("/myapp/mytargz.tar.gz", job); + * DistributedCache.addCacheArchive(new URI("/myapp/mytar.tar"), job); + * DistributedCache.addCacheArchive(new URI("/myapp/mytgz.tgz"), job); + * DistributedCache.addCacheArchive(new URI("/myapp/mytargz.tar.gz"), job); * * 3. Use the cached files in the {@link org.apache.hadoop.mapred.Mapper} * or {@link org.apache.hadoop.mapred.Reducer}: From 35034653d02ac8156338d7267e5975d2d66272d5 Mon Sep 17 00:00:00 2001 From: Varun Saxena Date: Mon, 20 Mar 2017 23:29:09 +0530 Subject: [PATCH 096/188] YARN-6353. Clean up OrderingPolicy javadoc (Daniel Templeton via Varun Saxena) --- .../scheduler/policy/OrderingPolicy.java | 105 ++++++++++-------- 1 file changed, 61 insertions(+), 44 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/OrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/OrderingPolicy.java index 1616bb1c79f..9aacc7e79a4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/OrderingPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/OrderingPolicy.java @@ -19,15 +19,13 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy; import java.util.*; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.*; /** * OrderingPolicy is used by the scheduler to order SchedulableEntities for - * container assignment and preemption + * container assignment and preemption. + * @param the type of {@link SchedulableEntity} that will be compared */ public interface OrderingPolicy { /* @@ -35,80 +33,99 @@ public interface OrderingPolicy { * synchronization of all use of the SchedulableEntity Collection and * Iterators for correctness and to avoid concurrent modification issues */ - + /** - * Get the collection of SchedulableEntities which are managed by this - * OrderingPolicy - should include processes returned by the Assignment and - * Preemption iterator with no guarantees regarding order + * Get the collection of {@link SchedulableEntity} Objects which are managed + * by this OrderingPolicy - should include processes returned by the + * Assignment and Preemption iterator with no guarantees regarding order. + * @return a collection of {@link SchedulableEntity} objects */ public Collection getSchedulableEntities(); - + /** - * Return an iterator over the collection of SchedulableEntities which orders - * them for container assignment + * Return an iterator over the collection of {@link SchedulableEntity} + * objects which orders them for container assignment. + * @return an iterator over the collection of {@link SchedulableEntity} + * objects */ public Iterator getAssignmentIterator(); - + /** - * Return an iterator over the collection of SchedulableEntities which orders - * them for preemption + * Return an iterator over the collection of {@link SchedulableEntity} + * objects which orders them for preemption. + * @return an iterator over the collection of {@link SchedulableEntity} */ public Iterator getPreemptionIterator(); - + /** - * Add a SchedulableEntity to be managed for allocation and preemption - * ordering + * Add a {@link SchedulableEntity} to be managed for allocation and preemption + * ordering. + * @param s the {@link SchedulableEntity} to add */ public void addSchedulableEntity(S s); - + /** - * Remove a SchedulableEntity from management for allocation and preemption - * ordering + * Remove a {@link SchedulableEntity} from management for allocation and + * preemption ordering. + * @param s the {@link SchedulableEntity} to remove + * @return whether the {@link SchedulableEntity} was present before this + * operation */ public boolean removeSchedulableEntity(S s); - + /** - * Add a collection of SchedulableEntities to be managed for allocation - * and preemption ordering + * Add a collection of {@link SchedulableEntity} objects to be managed for + * allocation and preemption ordering. + * @param sc the collection of {@link SchedulableEntity} objects to add */ public void addAllSchedulableEntities(Collection sc); - + /** - * Get the number of SchedulableEntities managed for allocation and - * preemption ordering + * Get the number of {@link SchedulableEntity} objects managed for allocation + * and preemption ordering. + * @return the number of {@link SchedulableEntity} objects */ public int getNumSchedulableEntities(); - + /** * Provides configuration information for the policy from the scheduler - * configuration + * configuration. + * @param conf a map of scheduler configuration properties and values */ public void configure(Map conf); - + /** - * The passed SchedulableEntity has been allocated the passed Container, - * take appropriate action (depending on comparator, a reordering of the - * SchedulableEntity may be required) + * Notify the {@code OrderingPolicy} that the {@link SchedulableEntity} + * has been allocated the given {@link RMContainer}, enabling the + * {@code OrderingPolicy} to take appropriate action. Depending on the + * comparator, a reordering of the {@link SchedulableEntity} may be required. + * @param schedulableEntity the {@link SchedulableEntity} + * @param r the allocated {@link RMContainer} */ - public void containerAllocated(S schedulableEntity, - RMContainer r); - + public void containerAllocated(S schedulableEntity, RMContainer r); + /** - * The passed SchedulableEntity has released the passed Container, - * take appropriate action (depending on comparator, a reordering of the - * SchedulableEntity may be required) + * Notify the {@code OrderingPolicy} that the {@link SchedulableEntity} + * has released the given {@link RMContainer}, enabling the + * {@code OrderingPolicy} to take appropriate action. Depending on the + * comparator, a reordering of the {@link SchedulableEntity} may be required. + * @param schedulableEntity the {@link SchedulableEntity} + * @param r the released {@link RMContainer} */ - public void containerReleased(S schedulableEntity, - RMContainer r); - + public void containerReleased(S schedulableEntity, RMContainer r); + /** - * Demand Updated for the passed schedulableEntity, reorder if needed. + * Notify the {@code OrderingPolicy} that the demand for the + * {@link SchedulableEntity} has been updated, enabling the + * {@code OrderingPolicy} to reorder the {@link SchedulableEntity} if needed. + * @param schedulableEntity the updated {@link SchedulableEntity} */ void demandUpdated(S schedulableEntity); /** - * Display information regarding configuration and status + * Return information regarding configuration and status. + * @return configuration and status information */ public String getInfo(); - + } From 6c399a88e9b5ef8f822a9bd469dbf9fdb3141e38 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Mon, 20 Mar 2017 11:29:23 -0700 Subject: [PATCH 097/188] HADOOP-14059. typo in s3a rename(self, subdir) error message. Contributed by Steve Loughran. --- .../src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 1786e68a53a..e0ff5a884ba 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -813,7 +813,7 @@ public class S3AFileSystem extends FileSystem { //Verify dest is not a child of the source directory if (dstKey.startsWith(srcKey)) { throw new RenameFailedException(srcKey, dstKey, - "cannot rename a directory to a subdirectory o fitself "); + "cannot rename a directory to a subdirectory of itself "); } List keysToDelete = new ArrayList<>(); From 49efd5d204524f49a8b91ece84c4131b2d49cf00 Mon Sep 17 00:00:00 2001 From: Daniel Templeton Date: Mon, 20 Mar 2017 14:23:55 -0700 Subject: [PATCH 098/188] YARN-6309. Fair scheduler docs should have the queue and queuePlacementPolicy elements listed in bold so that they're easier to see (Contributed by Esmaeil Mirzaee via Daniel Templeton) --- .../src/site/markdown/FairScheduler.md | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md index 2f0164d29c1..6661bc1d51d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md @@ -83,31 +83,31 @@ The allocation file must be in XML format. The format contains five types of ele * **Queue elements**: which represent queues. Queue elements can take an optional attribute 'type', which when set to 'parent' makes it a parent queue. This is useful when we want to create a parent queue without configuring any leaf queues. Each queue element may contain the following properties: - * minResources: minimum resources the queue is entitled to, in the form "X mb, Y vcores". For the single-resource fairness policy, the vcores value is ignored. If a queue's minimum share is not satisfied, it will be offered available resources before any other queue under the same parent. Under the single-resource fairness policy, a queue is considered unsatisfied if its memory usage is below its minimum memory share. Under dominant resource fairness, a queue is considered unsatisfied if its usage for its dominant resource with respect to the cluster capacity is below its minimum share for that resource. If multiple queues are unsatisfied in this situation, resources go to the queue with the smallest ratio between relevant resource usage and minimum. Note that it is possible that a queue that is below its minimum may not immediately get up to its minimum when it submits an application, because already-running jobs may be using those resources. + * **minResources**: minimum resources the queue is entitled to, in the form "X mb, Y vcores". For the single-resource fairness policy, the vcores value is ignored. If a queue's minimum share is not satisfied, it will be offered available resources before any other queue under the same parent. Under the single-resource fairness policy, a queue is considered unsatisfied if its memory usage is below its minimum memory share. Under dominant resource fairness, a queue is considered unsatisfied if its usage for its dominant resource with respect to the cluster capacity is below its minimum share for that resource. If multiple queues are unsatisfied in this situation, resources go to the queue with the smallest ratio between relevant resource usage and minimum. Note that it is possible that a queue that is below its minimum may not immediately get up to its minimum when it submits an application, because already-running jobs may be using those resources. - * maxResources: maximum resources a queue is allowed, in the form "X mb, Y vcores". A queue will never be assigned a container that would put its aggregate usage over this limit. + * **maxResources**: maximum resources a queue is allowed, in the form "X mb, Y vcores". A queue will never be assigned a container that would put its aggregate usage over this limit. - * maxChildResources: maximum resources an ad hoc child queue is allowed, in the form "X mb, Y vcores". Any ad hoc queue that is a direct child of a queue with this property set will have it's maxResources property set accordingly. + * **maxChildResources**: maximum resources an ad hoc child queue is allowed, in the form "X mb, Y vcores". Any ad hoc queue that is a direct child of a queue with this property set will have it's maxResources property set accordingly. - * maxRunningApps: limit the number of apps from the queue to run at once + * **maxRunningApps**: limit the number of apps from the queue to run at once - * maxAMShare: limit the fraction of the queue's fair share that can be used to run application masters. This property can only be used for leaf queues. For example, if set to 1.0f, then AMs in the leaf queue can take up to 100% of both the memory and CPU fair share. The value of -1.0f will disable this feature and the amShare will not be checked. The default value is 0.5f. + * **maxAMShare**: limit the fraction of the queue's fair share that can be used to run application masters. This property can only be used for leaf queues. For example, if set to 1.0f, then AMs in the leaf queue can take up to 100% of both the memory and CPU fair share. The value of -1.0f will disable this feature and the amShare will not be checked. The default value is 0.5f. - * weight: to share the cluster non-proportionally with other queues. Weights default to 1, and a queue with weight 2 should receive approximately twice as many resources as a queue with the default weight. + * **weight**: to share the cluster non-proportionally with other queues. Weights default to 1, and a queue with weight 2 should receive approximately twice as many resources as a queue with the default weight. - * schedulingPolicy: to set the scheduling policy of any queue. The allowed values are "fifo"/"fair"/"drf" or any class that extends `org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.SchedulingPolicy`. Defaults to "fair". If "fifo", apps with earlier submit times are given preference for containers, but apps submitted later may run concurrently if there is leftover space on the cluster after satisfying the earlier app's requests. + * **schedulingPolicy**: to set the scheduling policy of any queue. The allowed values are "fifo"/"fair"/"drf" or any class that extends `org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.SchedulingPolicy`. Defaults to "fair". If "fifo", apps with earlier submit times are given preference for containers, but apps submitted later may run concurrently if there is leftover space on the cluster after satisfying the earlier app's requests. - * aclSubmitApps: a list of users and/or groups that can submit apps to the queue. Refer to the ACLs section below for more info on the format of this list and how queue ACLs work. + * **aclSubmitApps**: a list of users and/or groups that can submit apps to the queue. Refer to the ACLs section below for more info on the format of this list and how queue ACLs work. - * aclAdministerApps: a list of users and/or groups that can administer a queue. Currently the only administrative action is killing an application. Refer to the ACLs section below for more info on the format of this list and how queue ACLs work. + * **aclAdministerApps**: a list of users and/or groups that can administer a queue. Currently the only administrative action is killing an application. Refer to the ACLs section below for more info on the format of this list and how queue ACLs work. - * minSharePreemptionTimeout: number of seconds the queue is under its minimum share before it will try to preempt containers to take resources from other queues. If not set, the queue will inherit the value from its parent queue. + * **minSharePreemptionTimeout**: number of seconds the queue is under its minimum share before it will try to preempt containers to take resources from other queues. If not set, the queue will inherit the value from its parent queue. - * fairSharePreemptionTimeout: number of seconds the queue is under its fair share threshold before it will try to preempt containers to take resources from other queues. If not set, the queue will inherit the value from its parent queue. + * **fairSharePreemptionTimeout**: number of seconds the queue is under its fair share threshold before it will try to preempt containers to take resources from other queues. If not set, the queue will inherit the value from its parent queue. - * fairSharePreemptionThreshold: the fair share preemption threshold for the queue. If the queue waits fairSharePreemptionTimeout without receiving fairSharePreemptionThreshold\*fairShare resources, it is allowed to preempt containers to take resources from other queues. If not set, the queue will inherit the value from its parent queue. + * **fairSharePreemptionThreshold**: the fair share preemption threshold for the queue. If the queue waits fairSharePreemptionTimeout without receiving fairSharePreemptionThreshold\*fairShare resources, it is allowed to preempt containers to take resources from other queues. If not set, the queue will inherit the value from its parent queue. - * allowPreemptionFrom: determines whether the scheduler is allowed to preempt resources from the queue. The default is true. If a queue has this property set to false, this property will apply recursively to all child queues. + * **allowPreemptionFrom**: determines whether the scheduler is allowed to preempt resources from the queue. The default is true. If a queue has this property set to false, this property will apply recursively to all child queues. * **User elements**: which represent settings governing the behavior of individual users. They can contain a single property: maxRunningApps, a limit on the number of running apps for a particular user. @@ -129,19 +129,19 @@ The allocation file must be in XML format. The format contains five types of ele * **A queuePlacementPolicy element**: which contains a list of rule elements that tell the scheduler how to place incoming apps into queues. Rules are applied in the order that they are listed. Rules may take arguments. All rules accept the "create" argument, which indicates whether the rule can create a new queue. "Create" defaults to true; if set to false and the rule would place the app in a queue that is not configured in the allocations file, we continue on to the next rule. The last rule must be one that can never issue a continue. Valid rules are: - * specified: the app is placed into the queue it requested. If the app requested no queue, i.e. it specified "default", we continue. If the app requested a queue name starting or ending with period, i.e. names like ".q1" or "q1." will be rejected. + * **specified**: the app is placed into the queue it requested. If the app requested no queue, i.e. it specified "default", we continue. If the app requested a queue name starting or ending with period, i.e. names like ".q1" or "q1." will be rejected. - * user: the app is placed into a queue with the name of the user who submitted it. Periods in the username will be replace with "\_dot\_", i.e. the queue name for user "first.last" is "first\_dot\_last". + * **user**: the app is placed into a queue with the name of the user who submitted it. Periods in the username will be replace with "\_dot\_", i.e. the queue name for user "first.last" is "first\_dot\_last". - * primaryGroup: the app is placed into a queue with the name of the primary group of the user who submitted it. Periods in the group name will be replaced with "\_dot\_", i.e. the queue name for group "one.two" is "one\_dot\_two". + * **primaryGroup**: the app is placed into a queue with the name of the primary group of the user who submitted it. Periods in the group name will be replaced with "\_dot\_", i.e. the queue name for group "one.two" is "one\_dot\_two". - * secondaryGroupExistingQueue: the app is placed into a queue with a name that matches a secondary group of the user who submitted it. The first secondary group that matches a configured queue will be selected. Periods in group names will be replaced with "\_dot\_", i.e. a user with "one.two" as one of their secondary groups would be placed into the "one\_dot\_two" queue, if such a queue exists. + * **secondaryGroupExistingQueue**: the app is placed into a queue with a name that matches a secondary group of the user who submitted it. The first secondary group that matches a configured queue will be selected. Periods in group names will be replaced with "\_dot\_", i.e. a user with "one.two" as one of their secondary groups would be placed into the "one\_dot\_two" queue, if such a queue exists. - * nestedUserQueue : the app is placed into a queue with the name of the user under the queue suggested by the nested rule. This is similar to ‘user’ rule,the difference being in 'nestedUserQueue' rule,user queues can be created under any parent queue, while 'user' rule creates user queues only under root queue. Note that nestedUserQueue rule would be applied only if the nested rule returns a parent queue.One can configure a parent queue either by setting 'type' attribute of queue to 'parent' or by configuring at least one leaf under that queue which makes it a parent. See example allocation for a sample use case. + * **nestedUserQueue**: the app is placed into a queue with the name of the user under the queue suggested by the nested rule. This is similar to ‘user’ rule,the difference being in 'nestedUserQueue' rule,user queues can be created under any parent queue, while 'user' rule creates user queues only under root queue. Note that nestedUserQueue rule would be applied only if the nested rule returns a parent queue.One can configure a parent queue either by setting 'type' attribute of queue to 'parent' or by configuring at least one leaf under that queue which makes it a parent. See example allocation for a sample use case. - * default: the app is placed into the queue specified in the 'queue' attribute of the default rule. If 'queue' attribute is not specified, the app is placed into 'root.default' queue. + * **default**: the app is placed into the queue specified in the 'queue' attribute of the default rule. If 'queue' attribute is not specified, the app is placed into 'root.default' queue. - * reject: the app is rejected. + * **reject**: the app is rejected. An example allocation file is given here: From 8e15e240597f821968e14893eabfea39815de207 Mon Sep 17 00:00:00 2001 From: Mingliang Liu Date: Mon, 20 Mar 2017 16:28:15 -0700 Subject: [PATCH 099/188] HADOOP-13945. Azure: Add Kerberos and Delegation token support to WASB client. Contributed by Santhosh G Nayak Change-Id: I39fb0a3b0491bf2160571366939a0502a0045429 --- .../fs/azure/AzureNativeFileSystemStore.java | 2 +- .../fs/azure/NativeAzureFileSystem.java | 67 +++++- .../fs/azure/RemoteSASKeyGeneratorImpl.java | 193 ++++++++++++------ .../fs/azure/RemoteWasbAuthorizerImpl.java | 147 ++++++++++--- .../fs/azure/SecureStorageInterfaceImpl.java | 11 +- .../fs/azure/WasbAuthorizerInterface.java | 5 +- .../hadoop/fs/azure/security/Constants.java | 54 +++++ .../fs/azure/security/SecurityUtils.java | 86 ++++++++ .../WasbDelegationTokenIdentifier.java | 48 +++++ .../fs/azure/security/WasbTokenRenewer.java | 150 ++++++++++++++ .../fs/azure/security/package-info.java | 22 ++ ...ache.hadoop.security.token.TokenIdentifier | 16 ++ ....apache.hadoop.security.token.TokenRenewer | 16 ++ .../fs/azure/MockWasbAuthorizerImpl.java | 4 +- 14 files changed, 714 insertions(+), 107 deletions(-) create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/Constants.java create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/SecurityUtils.java create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbDelegationTokenIdentifier.java create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbTokenRenewer.java create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/package-info.java create mode 100644 hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier create mode 100644 hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java index a8708ecafdf..a5bb3701728 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java @@ -478,7 +478,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore { this.storageInteractionLayer = new StorageInterfaceImpl(); } else { this.storageInteractionLayer = new SecureStorageInterfaceImpl( - useLocalSasKeyMode, conf, delegationToken); + useLocalSasKeyMode, conf); } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java index 9aebbb5a22e..1e6f00eb337 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java @@ -27,7 +27,9 @@ import java.io.InputStream; import java.io.OutputStream; import java.net.URI; import java.net.URISyntaxException; +import java.net.URL; import java.nio.charset.Charset; +import java.security.PrivilegedExceptionAction; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; @@ -61,10 +63,16 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation; import org.apache.hadoop.fs.azure.metrics.AzureFileSystemMetricsSystem; +import org.apache.hadoop.fs.azure.security.Constants; +import org.apache.hadoop.fs.azure.security.SecurityUtils; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator; +import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticator; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.Time; import org.slf4j.Logger; @@ -1107,6 +1115,9 @@ public class NativeAzureFileSystem extends FileSystem { // A counter to create unique (within-process) names for my metrics sources. private static AtomicInteger metricsSourceNameCounter = new AtomicInteger(); private boolean appendSupportEnabled = false; + private DelegationTokenAuthenticatedURL authURL; + private DelegationTokenAuthenticatedURL.Token authToken = new DelegationTokenAuthenticatedURL.Token(); + private String credServiceUrl; /** * Configuration key to enable authorization support in WASB. @@ -1124,6 +1135,11 @@ public class NativeAzureFileSystem extends FileSystem { */ private boolean azureAuthorization = false; + /** + * Flag controlling Kerberos support in WASB. + */ + private boolean kerberosSupportEnabled = false; + /** * Authorizer to use when authorization support is enabled in * WASB. @@ -1278,6 +1294,12 @@ public class NativeAzureFileSystem extends FileSystem { new RemoteWasbAuthorizerImpl(); authorizer.init(conf); } + + if (UserGroupInformation.isSecurityEnabled() && kerberosSupportEnabled) { + DelegationTokenAuthenticator authenticator = new KerberosDelegationTokenAuthenticator(); + authURL = new DelegationTokenAuthenticatedURL(authenticator); + credServiceUrl = SecurityUtils.getCredServiceUrls(conf); + } } @Override @@ -1406,7 +1428,7 @@ public class NativeAzureFileSystem extends FileSystem { String operation) throws WasbAuthorizationException, IOException { if (azureAuthorization && this.authorizer != null && - !this.authorizer.authorize(path, accessType, delegationToken)) { + !this.authorizer.authorize(path, accessType)) { throw new WasbAuthorizationException(operation + " operation for Path : " + path + " not allowed"); } @@ -2899,6 +2921,49 @@ public class NativeAzureFileSystem extends FileSystem { isClosed = true; } + /** + * Get a delegation token from remote service endpoint if + * 'fs.azure.enable.kerberos.support' is set to 'true'. + * @param renewer the account name that is allowed to renew the token. + * @return delegation token + * @throws IOException thrown when getting the current user. + */ + @Override + public Token getDelegationToken(final String renewer) throws IOException { + if (kerberosSupportEnabled) { + try { + final UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + UserGroupInformation connectUgi = ugi.getRealUser(); + final UserGroupInformation proxyUser = connectUgi; + if (connectUgi == null) { + connectUgi = ugi; + } + if (!connectUgi.hasKerberosCredentials()) { + connectUgi = UserGroupInformation.getLoginUser(); + } + connectUgi.checkTGTAndReloginFromKeytab(); + return connectUgi.doAs(new PrivilegedExceptionAction>() { + @Override + public Token run() throws Exception { + return authURL.getDelegationToken(new URL(credServiceUrl + + Constants.DEFAULT_DELEGATION_TOKEN_MANAGER_ENDPOINT), + authToken, renewer, (proxyUser != null)? ugi.getShortUserName(): null); + } + }); + } catch (Exception ex) { + LOG.error("Error in fetching the delegation token from remote service", + ex); + if (ex instanceof IOException) { + throw (IOException) ex; + } else { + throw new IOException(ex); + } + } + } else { + return super.getDelegationToken(renewer); + } + } + /** * A handler that defines what to do with blobs whose upload was * interrupted. diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteSASKeyGeneratorImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteSASKeyGeneratorImpl.java index 404419d7422..aab62a14dcc 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteSASKeyGeneratorImpl.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteSASKeyGeneratorImpl.java @@ -21,10 +21,22 @@ package org.apache.hadoop.fs.azure; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.net.UnknownHostException; +import java.security.PrivilegedExceptionAction; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.Validate; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.azure.security.Constants; +import org.apache.hadoop.fs.azure.security.SecurityUtils; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.client.AuthenticatedURL; +import org.apache.hadoop.security.authentication.client.AuthenticationException; +import org.apache.hadoop.security.authentication.client.Authenticator; +import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticator; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.utils.URIBuilder; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,12 +55,6 @@ public class RemoteSASKeyGeneratorImpl extends SASKeyGeneratorImpl { public static final Logger LOG = LoggerFactory.getLogger(AzureNativeFileSystemStore.class); - /** - * Configuration parameter name expected in the Configuration - * object to provide the url of the remote service {@value} - */ - private static final String KEY_CRED_SERVICE_URL = - "fs.azure.cred.service.url"; /** * Container SAS Key generation OP name. {@value} @@ -82,7 +88,7 @@ public class RemoteSASKeyGeneratorImpl extends SASKeyGeneratorImpl { * Query parameter name for user info {@value} */ private static final String DELEGATION_TOKEN_QUERY_PARAM_NAME = - "delegation_token"; + "delegation"; /** * Query parameter name for the relative path inside the storage @@ -94,41 +100,50 @@ public class RemoteSASKeyGeneratorImpl extends SASKeyGeneratorImpl { private String delegationToken = ""; private String credServiceUrl = ""; private WasbRemoteCallHelper remoteCallHelper = null; + private boolean isSecurityEnabled; + private boolean isKerberosSupportEnabled; public RemoteSASKeyGeneratorImpl(Configuration conf) { super(conf); } - public boolean initialize(Configuration conf, String delegationToken) { + public void initialize(Configuration conf) throws IOException { LOG.debug("Initializing RemoteSASKeyGeneratorImpl instance"); - credServiceUrl = conf.get(KEY_CRED_SERVICE_URL); - - if (delegationToken == null || delegationToken.isEmpty()) { - LOG.error("Delegation Token not provided for initialization" - + " of RemoteSASKeyGenerator"); - return false; + try { + delegationToken = SecurityUtils.getDelegationTokenFromCredentials(); + } catch (IOException e) { + final String msg = "Error in fetching the WASB delegation token"; + LOG.error(msg, e); + throw new IOException(msg, e); } - this.delegationToken = delegationToken; + try { + credServiceUrl = SecurityUtils.getCredServiceUrls(conf); + } catch (UnknownHostException e) { + final String msg = "Invalid CredService Url, configure it correctly"; + LOG.error(msg, e); + throw new IOException(msg, e); + } if (credServiceUrl == null || credServiceUrl.isEmpty()) { - LOG.error("CredService Url not found in configuration to initialize" - + " RemoteSASKeyGenerator"); - return false; + final String msg = "CredService Url not found in configuration to " + + "initialize RemoteSASKeyGenerator"; + LOG.error(msg); + throw new IOException(msg); } remoteCallHelper = new WasbRemoteCallHelper(); - LOG.debug("Initialization of RemoteSASKeyGenerator instance successfull"); - return true; + this.isSecurityEnabled = UserGroupInformation.isSecurityEnabled(); + this.isKerberosSupportEnabled = conf.getBoolean( + Constants.AZURE_KERBEROS_SUPPORT_PROPERTY_NAME, false); + LOG.debug("Initialization of RemoteSASKeyGenerator instance successful"); } @Override public URI getContainerSASUri(String storageAccount, String container) throws SASKeyGenerationException { - try { - LOG.debug("Generating Container SAS Key for Container {} " + "inside Storage Account {} ", container, storageAccount); URIBuilder uriBuilder = new URIBuilder(credServiceUrl); @@ -139,38 +154,39 @@ public class RemoteSASKeyGeneratorImpl extends SASKeyGeneratorImpl { container); uriBuilder.addParameter(SAS_EXPIRY_QUERY_PARAM_NAME, "" + getSasKeyExpiryPeriod()); - uriBuilder.addParameter(DELEGATION_TOKEN_QUERY_PARAM_NAME, - this.delegationToken); - - RemoteSASKeyGenerationResponse sasKeyResponse = - makeRemoteRequest(uriBuilder.build()); - - if (sasKeyResponse == null) { - throw new SASKeyGenerationException("RemoteSASKeyGenerationResponse" - + " object null from remote call"); - } else if (sasKeyResponse.getResponseCode() - == REMOTE_CALL_SUCCESS_CODE) { - return new URI(sasKeyResponse.getSasKey()); - } else { - throw new SASKeyGenerationException("Remote Service encountered error" - + " in SAS Key generation : " - + sasKeyResponse.getResponseMessage()); + if (isSecurityEnabled && StringUtils.isNotEmpty(delegationToken)) { + uriBuilder.addParameter(DELEGATION_TOKEN_QUERY_PARAM_NAME, + this.delegationToken); } + + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + UserGroupInformation connectUgi = ugi.getRealUser(); + if (connectUgi == null) { + connectUgi = ugi; + } else { + uriBuilder.addParameter(Constants.DOAS_PARAM, ugi.getShortUserName()); + } + + if (isSecurityEnabled && !connectUgi.hasKerberosCredentials()) { + connectUgi = UserGroupInformation.getLoginUser(); + } + return getSASKey(uriBuilder.build(), connectUgi); } catch (URISyntaxException uriSyntaxEx) { throw new SASKeyGenerationException("Encountered URISyntaxException " + "while building the HttpGetRequest to remote cred service", uriSyntaxEx); + } catch (IOException e) { + throw new SASKeyGenerationException("Encountered IOException" + + " while building the HttpGetRequest to remote service", e); } } @Override public URI getRelativeBlobSASUri(String storageAccount, String container, String relativePath) throws SASKeyGenerationException { - try { - LOG.debug("Generating RelativePath SAS Key for relativePath {} inside" - + " Container {} inside Storage Account {} ", + + " Container {} inside Storage Account {} ", relativePath, container, storageAccount); URIBuilder uriBuilder = new URIBuilder(credServiceUrl); uriBuilder.setPath("/" + BLOB_SAS_OP); @@ -182,41 +198,98 @@ public class RemoteSASKeyGeneratorImpl extends SASKeyGeneratorImpl { relativePath); uriBuilder.addParameter(SAS_EXPIRY_QUERY_PARAM_NAME, "" + getSasKeyExpiryPeriod()); - uriBuilder.addParameter(DELEGATION_TOKEN_QUERY_PARAM_NAME, - this.delegationToken); - RemoteSASKeyGenerationResponse sasKeyResponse = - makeRemoteRequest(uriBuilder.build()); - - if (sasKeyResponse == null) { - throw new SASKeyGenerationException("RemoteSASKeyGenerationResponse" - + " object null from remote call"); - } else if (sasKeyResponse.getResponseCode() - == REMOTE_CALL_SUCCESS_CODE) { - return new URI(sasKeyResponse.getSasKey()); - } else { - throw new SASKeyGenerationException("Remote Service encountered error" - + " in SAS Key generation : " - + sasKeyResponse.getResponseMessage()); + if (isSecurityEnabled && StringUtils.isNotEmpty( + delegationToken)) { + uriBuilder.addParameter(DELEGATION_TOKEN_QUERY_PARAM_NAME, + this.delegationToken); } + + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + UserGroupInformation connectUgi = ugi.getRealUser(); + if (connectUgi == null) { + connectUgi = ugi; + } else { + uriBuilder.addParameter(Constants.DOAS_PARAM, ugi.getShortUserName()); + } + + if (isSecurityEnabled && !connectUgi.hasKerberosCredentials()) { + connectUgi = UserGroupInformation.getLoginUser(); + } + return getSASKey(uriBuilder.build(), connectUgi); } catch (URISyntaxException uriSyntaxEx) { throw new SASKeyGenerationException("Encountered URISyntaxException" + " while building the HttpGetRequest to " + " remote service", uriSyntaxEx); + } catch (IOException e) { + throw new SASKeyGenerationException("Encountered IOException" + + " while building the HttpGetRequest to remote service", e); + } + } + + private URI getSASKey(final URI uri, UserGroupInformation connectUgi) + throws URISyntaxException, SASKeyGenerationException { + final RemoteSASKeyGenerationResponse sasKeyResponse; + try { + connectUgi.checkTGTAndReloginFromKeytab(); + sasKeyResponse = connectUgi.doAs( + new PrivilegedExceptionAction() { + @Override + public RemoteSASKeyGenerationResponse run() throws Exception { + AuthenticatedURL.Token token = null; + if (isKerberosSupportEnabled && UserGroupInformation + .isSecurityEnabled() && (delegationToken == null + || delegationToken.isEmpty())) { + token = new AuthenticatedURL.Token(); + final Authenticator kerberosAuthenticator = + new KerberosDelegationTokenAuthenticator(); + try { + kerberosAuthenticator.authenticate(uri.toURL(), token); + Validate.isTrue(token.isSet(), + "Authenticated Token is NOT present. " + + "The request cannot proceed."); + } catch (AuthenticationException e) { + throw new IOException( + "Authentication failed in check authorization", e); + } + } + return makeRemoteRequest(uri, + (token != null ? token.toString() : null)); + } + }); + } catch (InterruptedException | IOException e) { + final String msg = "Error fetching SAS Key from Remote Service: " + uri; + LOG.error(msg, e); + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw new SASKeyGenerationException(msg, e); + } + + if (sasKeyResponse.getResponseCode() == REMOTE_CALL_SUCCESS_CODE) { + return new URI(sasKeyResponse.getSasKey()); + } else { + throw new SASKeyGenerationException( + "Remote Service encountered error in SAS Key generation : " + + sasKeyResponse.getResponseMessage()); } } /** * Helper method to make a remote request. * @param uri - Uri to use for the remote request + * @param token - hadoop.auth token for the remote request * @return RemoteSASKeyGenerationResponse */ - private RemoteSASKeyGenerationResponse makeRemoteRequest(URI uri) - throws SASKeyGenerationException { + private RemoteSASKeyGenerationResponse makeRemoteRequest(URI uri, + String token) throws SASKeyGenerationException { try { - String responseBody = - remoteCallHelper.makeRemoteGetRequest(new HttpGet(uri)); + HttpGet httpGet = new HttpGet(uri); + if (token != null) { + httpGet.setHeader("Cookie", AuthenticatedURL.AUTH_COOKIE + "=" + token); + } + String responseBody = remoteCallHelper.makeRemoteGetRequest(httpGet); ObjectMapper objectMapper = new ObjectMapper(); return objectMapper.readValue(responseBody, diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java index 5f2265bc732..a2105c7f8c3 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/RemoteWasbAuthorizerImpl.java @@ -22,12 +22,27 @@ import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.Validate; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.azure.security.Constants; +import org.apache.hadoop.fs.azure.security.SecurityUtils; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.client.AuthenticatedURL; +import org.apache.hadoop.security.authentication.client.AuthenticationException; +import org.apache.hadoop.security.authentication.client.Authenticator; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticator; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.utils.URIBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.net.URISyntaxException; +import java.security.PrivilegedExceptionAction; +import java.util.Iterator; import static org.apache.hadoop.fs.azure.WasbRemoteCallHelper.REMOTE_CALL_SUCCESS_CODE; @@ -39,7 +54,10 @@ import static org.apache.hadoop.fs.azure.WasbRemoteCallHelper.REMOTE_CALL_SUCCES */ public class RemoteWasbAuthorizerImpl implements WasbAuthorizerInterface { - private String remoteAuthorizerServiceUrl = ""; + public static final Logger LOG = LoggerFactory + .getLogger(RemoteWasbAuthorizerImpl.class); + + private String remoteAuthorizerServiceUrl = null; /** * Configuration parameter name expected in the Configuration object to @@ -70,9 +88,12 @@ public class RemoteWasbAuthorizerImpl implements WasbAuthorizerInterface { * Query parameter name for user info {@value} */ private static final String DELEGATION_TOKEN_QUERY_PARAM_NAME = - "delegation_token"; + "delegation"; private WasbRemoteCallHelper remoteCallHelper = null; + private String delegationToken; + private boolean isSecurityEnabled; + private boolean isKerberosSupportEnabled; @VisibleForTesting public void updateWasbRemoteCallHelper(WasbRemoteCallHelper helper) { @@ -82,55 +103,113 @@ public class RemoteWasbAuthorizerImpl implements WasbAuthorizerInterface { @Override public void init(Configuration conf) throws WasbAuthorizationException, IOException { + LOG.debug("Initializing RemoteWasbAuthorizerImpl instance"); + Iterator> tokenIterator = null; + try { + delegationToken = SecurityUtils.getDelegationTokenFromCredentials(); + } catch (IOException e) { + final String msg = "Error in fetching the WASB delegation token"; + LOG.error(msg, e); + throw new IOException(msg, e); + } - remoteAuthorizerServiceUrl = conf.get(KEY_REMOTE_AUTH_SERVICE_URL); + remoteAuthorizerServiceUrl = SecurityUtils + .getRemoteAuthServiceUrls(conf); if (remoteAuthorizerServiceUrl == null || remoteAuthorizerServiceUrl.isEmpty()) { throw new WasbAuthorizationException( "fs.azure.authorization.remote.service.url config not set" - + " in configuration."); + + " in configuration."); } this.remoteCallHelper = new WasbRemoteCallHelper(); + this.isSecurityEnabled = UserGroupInformation.isSecurityEnabled(); + this.isKerberosSupportEnabled = conf + .getBoolean(Constants.AZURE_KERBEROS_SUPPORT_PROPERTY_NAME, false); } @Override - public boolean authorize(String wasbAbsolutePath, String accessType, - String delegationToken) throws WasbAuthorizationException, IOException { + public boolean authorize(String wasbAbsolutePath, String accessType) + throws WasbAuthorizationException, IOException { + try { + URIBuilder uriBuilder = new URIBuilder(remoteAuthorizerServiceUrl); + uriBuilder.setPath("/" + CHECK_AUTHORIZATION_OP); + uriBuilder.addParameter(WASB_ABSOLUTE_PATH_QUERY_PARAM_NAME, + wasbAbsolutePath); + uriBuilder.addParameter(ACCESS_OPERATION_QUERY_PARAM_NAME, + accessType); + if (isSecurityEnabled && StringUtils.isNotEmpty(delegationToken)) { + uriBuilder.addParameter(DELEGATION_TOKEN_QUERY_PARAM_NAME, + delegationToken); + } - try { - URIBuilder uriBuilder = new URIBuilder(remoteAuthorizerServiceUrl); - uriBuilder.setPath("/" + CHECK_AUTHORIZATION_OP); - uriBuilder.addParameter(WASB_ABSOLUTE_PATH_QUERY_PARAM_NAME, - wasbAbsolutePath); - uriBuilder.addParameter(ACCESS_OPERATION_QUERY_PARAM_NAME, - accessType); - uriBuilder.addParameter(DELEGATION_TOKEN_QUERY_PARAM_NAME, - delegationToken); + String responseBody = null; + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + UserGroupInformation connectUgi = ugi.getRealUser(); + if (connectUgi == null) { + connectUgi = ugi; + } else { + uriBuilder.addParameter(Constants.DOAS_PARAM, ugi.getShortUserName()); + } + if (isSecurityEnabled && !connectUgi.hasKerberosCredentials()) { + connectUgi = UserGroupInformation.getLoginUser(); + } + connectUgi.checkTGTAndReloginFromKeytab(); - String responseBody = remoteCallHelper.makeRemoteGetRequest( - new HttpGet(uriBuilder.build())); + try { + responseBody = connectUgi + .doAs(new PrivilegedExceptionAction() { + @Override + public String run() throws Exception { + AuthenticatedURL.Token token = null; + HttpGet httpGet = new HttpGet(uriBuilder.build()); + if (isKerberosSupportEnabled && UserGroupInformation + .isSecurityEnabled() && (delegationToken == null + || delegationToken.isEmpty())) { + token = new AuthenticatedURL.Token(); + final Authenticator kerberosAuthenticator = new KerberosDelegationTokenAuthenticator(); + try { + kerberosAuthenticator + .authenticate(uriBuilder.build().toURL(), token); + Validate.isTrue(token.isSet(), + "Authenticated Token is NOT present. The request cannot proceed."); + } catch (AuthenticationException e){ + throw new IOException("Authentication failed in check authorization", e); + } + if (token != null) { + httpGet.setHeader("Cookie", + AuthenticatedURL.AUTH_COOKIE + "=" + token); + } + } + return remoteCallHelper.makeRemoteGetRequest(httpGet); + } + }); + } catch (InterruptedException e) { + LOG.error("Error in check authorization", e); + throw new WasbAuthorizationException("Error in check authorize", e); + } - ObjectMapper objectMapper = new ObjectMapper(); - RemoteAuthorizerResponse authorizerResponse = - objectMapper.readValue(responseBody, RemoteAuthorizerResponse.class); + ObjectMapper objectMapper = new ObjectMapper(); + RemoteAuthorizerResponse authorizerResponse = + objectMapper + .readValue(responseBody, RemoteAuthorizerResponse.class); - if (authorizerResponse == null) { - throw new WasbAuthorizationException( - "RemoteAuthorizerResponse object null from remote call"); - } else if (authorizerResponse.getResponseCode() - == REMOTE_CALL_SUCCESS_CODE) { - return authorizerResponse.getAuthorizationResult(); - } else { - throw new WasbAuthorizationException("Remote authorization" - + " service encountered an error " - + authorizerResponse.getResponseMessage()); + if (authorizerResponse == null) { + throw new WasbAuthorizationException( + "RemoteAuthorizerResponse object null from remote call"); + } else if (authorizerResponse.getResponseCode() + == REMOTE_CALL_SUCCESS_CODE) { + return authorizerResponse.getAuthorizationResult(); + } else { + throw new WasbAuthorizationException("Remote authorization" + + " serivce encountered an error " + + authorizerResponse.getResponseMessage()); + } + } catch (URISyntaxException | WasbRemoteCallException + | JsonParseException | JsonMappingException ex) { + throw new WasbAuthorizationException(ex); } - } catch (URISyntaxException | WasbRemoteCallException - | JsonParseException | JsonMappingException ex) { - throw new WasbAuthorizationException(ex); - } } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java index 6749a76f7f4..650149af6ac 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SecureStorageInterfaceImpl.java @@ -69,21 +69,20 @@ public class SecureStorageInterfaceImpl extends StorageInterface { public static final String SAS_ERROR_CODE = "SAS Error"; private SASKeyGeneratorInterface sasKeyGenerator; private String storageAccount; - private String delegationToken; public SecureStorageInterfaceImpl(boolean useLocalSASKeyMode, - Configuration conf, String delegationToken) - throws SecureModeException { + Configuration conf) throws SecureModeException { - this.delegationToken = delegationToken; if (useLocalSASKeyMode) { this.sasKeyGenerator = new LocalSASKeyGeneratorImpl(conf); } else { RemoteSASKeyGeneratorImpl remoteSasKeyGenerator = new RemoteSASKeyGeneratorImpl(conf); - if (!remoteSasKeyGenerator.initialize(conf, this.delegationToken)) { + try { + remoteSasKeyGenerator.initialize(conf); + } catch (IOException ioe) { throw new SecureModeException("Remote SAS Key mode could" - + " not be initialized"); + + " not be initialized", ioe); } this.sasKeyGenerator = remoteSasKeyGenerator; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizerInterface.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizerInterface.java index f391851095a..57d75163c9f 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizerInterface.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbAuthorizerInterface.java @@ -43,11 +43,10 @@ public interface WasbAuthorizerInterface { * @param wasbAbolutePath : Absolute WASB Path used for access. * @param accessType : Type of access - * @param delegationToken : The user information. * @return : true - If access allowed false - If access is not allowed. * @throws WasbAuthorizationException - On authorization exceptions * @throws IOException - When not able to reach the authorizer */ - public boolean authorize(String wasbAbolutePath, String accessType, - String delegationToken) throws WasbAuthorizationException, IOException; + boolean authorize(String wasbAbolutePath, String accessType) + throws WasbAuthorizationException, IOException; } \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/Constants.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/Constants.java new file mode 100644 index 00000000000..79531a968f3 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/Constants.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure.security; + +/** + * Constants for used with WASB security implementation. + */ +public final class Constants { + + private Constants() { + } + + /** + * Configuration parameter name expected in the Configuration + * object to provide the url of the remote service {@value} + */ + public static final String KEY_CRED_SERVICE_URL = "fs.azure.cred.service.url"; + /** + * Default port of the remote service used as delegation token manager and Azure storage SAS key generator. + */ + public static final int DEFAULT_CRED_SERVICE_PORT = 50911; + + /** + * Default remote delegation token manager endpoint. + */ + public static final String DEFAULT_DELEGATION_TOKEN_MANAGER_ENDPOINT = "/tokenmanager/v1"; + + /** + * The configuration property to enable Kerberos support. + */ + + public static final String AZURE_KERBEROS_SUPPORT_PROPERTY_NAME = "fs.azure.enable.kerberos.support"; + + /** + * Parameter to be used for impersonation. + */ + public static final String DOAS_PARAM = "doas"; +} diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/SecurityUtils.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/SecurityUtils.java new file mode 100644 index 00000000000..61bf8461b2d --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/SecurityUtils.java @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure.security; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.azure.RemoteWasbAuthorizerImpl; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Iterator; + +/** + * Security Utils class for WASB. + */ +public final class SecurityUtils { + + private SecurityUtils() { + } + + /** + * Utility method to get remote service URLs from the configuration. + * @param conf configuration object. + * @return remote service URL + * @throws UnknownHostException thrown when getting the default value. + */ + public static String getCredServiceUrls(Configuration conf) + throws UnknownHostException { + return conf.get(Constants.KEY_CRED_SERVICE_URL, String + .format("http://%s:%s", + InetAddress.getLocalHost().getCanonicalHostName(), + Constants.DEFAULT_CRED_SERVICE_PORT)); + } + + /** + * Utility method to get remote Authorization service URLs from the configuration. + * @param conf Configuration object. + * @return remote Authorization server URL + * @throws UnknownHostException thrown when getting the default value. + */ + public static String getRemoteAuthServiceUrls(Configuration conf) + throws UnknownHostException { + return conf.get(RemoteWasbAuthorizerImpl.KEY_REMOTE_AUTH_SERVICE_URL, String + .format("http://%s:%s", + InetAddress.getLocalHost().getCanonicalHostName(), + Constants.DEFAULT_CRED_SERVICE_PORT)); + } + + /** + * Utility method to get delegation token from the UGI credentials. + * @return delegation token + * @throws IOException thrown when getting the current user. + */ + public static String getDelegationTokenFromCredentials() throws IOException { + String delegationToken = null; + Iterator> tokenIterator = UserGroupInformation + .getCurrentUser().getCredentials().getAllTokens().iterator(); + while (tokenIterator.hasNext()) { + Token iteratedToken = tokenIterator.next(); + if (iteratedToken.getKind() + .equals(WasbDelegationTokenIdentifier.TOKEN_KIND)) { + delegationToken = iteratedToken.encodeToUrlString(); + } + } + return delegationToken; + } +} diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbDelegationTokenIdentifier.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbDelegationTokenIdentifier.java new file mode 100644 index 00000000000..530e04572e2 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbDelegationTokenIdentifier.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure.security; + +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier; + +/** + * Delegation token Identifier for WASB delegation tokens. + */ +public class WasbDelegationTokenIdentifier extends DelegationTokenIdentifier { + public static final Text TOKEN_KIND = new Text("WASB delegation"); + + public WasbDelegationTokenIdentifier(){ + super(TOKEN_KIND); + } + + public WasbDelegationTokenIdentifier(Text kind) { + super(kind); + } + + public WasbDelegationTokenIdentifier(Text kind, Text owner, Text renewer, + Text realUser) { + super(kind, owner, renewer, realUser); + } + + @Override + public Text getKind() { + return TOKEN_KIND; + } + +} diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbTokenRenewer.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbTokenRenewer.java new file mode 100644 index 00000000000..642f56ac480 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/WasbTokenRenewer.java @@ -0,0 +1,150 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure.security; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenRenewer; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator; +import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.URL; +import java.security.PrivilegedExceptionAction; + +/** + * Token Renewer for renewing WASB delegation tokens with remote service. + */ +public class WasbTokenRenewer extends TokenRenewer { + public static final Logger LOG = LoggerFactory + .getLogger(WasbTokenRenewer.class); + + /** + * Checks if this particular object handles the Kind of token passed. + * @param kind the kind of the token + * @return true if it handles passed token kind false otherwise. + */ + @Override + public boolean handleKind(Text kind) { + return WasbDelegationTokenIdentifier.TOKEN_KIND.equals(kind); + } + + /** + * Checks if passed token is managed. + * @param token the token being checked + * @return true if it is managed. + * @throws IOException thrown when evaluating if token is managed. + */ + @Override + public boolean isManaged(Token token) throws IOException { + return true; + } + + /** + * Renew the delegation token. + * @param token token to renew. + * @param conf configuration object. + * @return extended expiry time of the token. + * @throws IOException thrown when trying get current user. + * @throws InterruptedException thrown when thread is interrupted + */ + @Override + public long renew(final Token token, Configuration conf) + throws IOException, InterruptedException { + LOG.debug("Renewing the delegation token"); + final UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + UserGroupInformation connectUgi = ugi.getRealUser(); + final UserGroupInformation proxyUser = connectUgi; + if (connectUgi == null) { + connectUgi = ugi; + } + if (!connectUgi.hasKerberosCredentials()) { + connectUgi = UserGroupInformation.getLoginUser(); + } + connectUgi.checkTGTAndReloginFromKeytab(); + final DelegationTokenAuthenticatedURL.Token authToken = new DelegationTokenAuthenticatedURL.Token(); + authToken + .setDelegationToken((Token) token); + final String credServiceUrl = conf.get(Constants.KEY_CRED_SERVICE_URL, + String.format("http://%s:%s", + InetAddress.getLocalHost().getCanonicalHostName(), + Constants.DEFAULT_CRED_SERVICE_PORT)); + DelegationTokenAuthenticator authenticator = new KerberosDelegationTokenAuthenticator(); + final DelegationTokenAuthenticatedURL authURL = new DelegationTokenAuthenticatedURL( + authenticator); + + return connectUgi.doAs(new PrivilegedExceptionAction() { + @Override + public Long run() throws Exception { + return authURL.renewDelegationToken(new URL(credServiceUrl + + Constants.DEFAULT_DELEGATION_TOKEN_MANAGER_ENDPOINT), + authToken, (proxyUser != null) ? ugi.getShortUserName() : null); + } + }); + } + + /** + * Cancel the delegation token. + * @param token token to cancel. + * @param conf configuration object. + * @throws IOException thrown when trying get current user. + * @throws InterruptedException thrown when thread is interrupted. + */ + @Override + public void cancel(final Token token, Configuration conf) + throws IOException, InterruptedException { + LOG.debug("Cancelling the delegation token"); + final UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + UserGroupInformation connectUgi = ugi.getRealUser(); + final UserGroupInformation proxyUser = connectUgi; + if (connectUgi == null) { + connectUgi = ugi; + } + if (!connectUgi.hasKerberosCredentials()) { + connectUgi = UserGroupInformation.getLoginUser(); + } + connectUgi.checkTGTAndReloginFromKeytab(); + final DelegationTokenAuthenticatedURL.Token authToken = new DelegationTokenAuthenticatedURL.Token(); + authToken + .setDelegationToken((Token) token); + final String credServiceUrl = conf.get(Constants.KEY_CRED_SERVICE_URL, + String.format("http://%s:%s", + InetAddress.getLocalHost().getCanonicalHostName(), + Constants.DEFAULT_CRED_SERVICE_PORT)); + DelegationTokenAuthenticator authenticator = new KerberosDelegationTokenAuthenticator(); + final DelegationTokenAuthenticatedURL authURL = new DelegationTokenAuthenticatedURL( + authenticator); + connectUgi.doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + authURL.cancelDelegationToken(new URL(credServiceUrl + + Constants.DEFAULT_DELEGATION_TOKEN_MANAGER_ENDPOINT), + authToken, (proxyUser != null) ? ugi.getShortUserName() : null); + return null; + } + }); + } +} \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/package-info.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/package-info.java new file mode 100644 index 00000000000..1e1bfbe3aea --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/security/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +package org.apache.hadoop.fs.azure.security; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier b/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier new file mode 100644 index 00000000000..7ec8216deb0 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.hadoop.fs.azure.security.WasbDelegationTokenIdentifier \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer b/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer new file mode 100644 index 00000000000..f9c590aad8d --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.hadoop.fs.azure.security.WasbTokenRenewer \ No newline at end of file diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java index 8f7cb2ae5ea..af5a537ce4d 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockWasbAuthorizerImpl.java @@ -44,8 +44,8 @@ public class MockWasbAuthorizerImpl implements WasbAuthorizerInterface { } @Override - public boolean authorize(String wasbAbsolutePath, String accessType, - String delegationToken) throws WasbAuthorizationException { + public boolean authorize(String wasbAbsolutePath, String accessType) + throws WasbAuthorizationException { AuthorizationComponent component = new AuthorizationComponent(wasbAbsolutePath, accessType); From b104f3a282ab8e15314446e71189eb3a3b5dd9a6 Mon Sep 17 00:00:00 2001 From: Chris Douglas Date: Mon, 20 Mar 2017 17:15:13 -0700 Subject: [PATCH 100/188] HDFS-6648. Order of namenodes in ConfiguredFailoverProxyProvider is undefined. Contributed by Inigo Goiri --- .../org/apache/hadoop/hdfs/DFSUtilClient.java | 2 +- .../hadoop/hdfs/client/HdfsClientConfigKeys.java | 2 ++ .../ha/ConfiguredFailoverProxyProvider.java | 15 ++++++++++++--- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java index d2675301065..f9b2e8d7c8a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java @@ -366,7 +366,7 @@ public class DFSUtilClient { static Map getAddressesForNameserviceId( Configuration conf, String nsId, String defaultValue, String... keys) { Collection nnIds = getNameNodeIds(conf, nsId); - Map ret = Maps.newHashMap(); + Map ret = Maps.newLinkedHashMap(); for (String nnId : emptyAsSingletonNull(nnIds)) { String suffix = concatSuffixes(nsId, nnId); String address = getConfValue(defaultValue, suffix, conf, keys); diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java index 6f8c661a5ba..1a388061ceb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java @@ -264,6 +264,8 @@ public interface HdfsClientConfigKeys { String CONNECTION_RETRIES_ON_SOCKET_TIMEOUTS_KEY = PREFIX + "connection.retries.on.timeouts"; int CONNECTION_RETRIES_ON_SOCKET_TIMEOUTS_DEFAULT = 0; + String RANDOM_ORDER = PREFIX + "random.order"; + boolean RANDOM_ORDER_DEFAULT = false; } /** dfs.client.write configuration properties */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java index c2d4d916261..0e8fa448806 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java @@ -23,6 +23,7 @@ import java.net.InetSocketAddress; import java.net.URI; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; @@ -43,9 +44,10 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; /** - * A FailoverProxyProvider implementation which allows one to configure two URIs - * to connect to during fail-over. The first configured address is tried first, - * and on a fail-over event the other address is tried. + * A FailoverProxyProvider implementation which allows one to configure + * multiple URIs to connect to during fail-over. A random configured address is + * tried first, and on a fail-over event the other addresses are tried + * sequentially in a random order. */ public class ConfiguredFailoverProxyProvider extends AbstractNNFailoverProxyProvider { @@ -124,6 +126,13 @@ public class ConfiguredFailoverProxyProvider extends for (InetSocketAddress address : addressesOfNns) { proxies.add(new AddressRpcProxyPair(address)); } + // Randomize the list to prevent all clients pointing to the same one + boolean randomized = conf.getBoolean( + HdfsClientConfigKeys.Failover.RANDOM_ORDER, + HdfsClientConfigKeys.Failover.RANDOM_ORDER_DEFAULT); + if (randomized) { + Collections.shuffle(proxies); + } // The client may have a delegation token set for the logical // URI of the cluster. Clone this token to apply to each of the From e7c8da614c37e36fb8081234f4c639d6054f6082 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Mon, 20 Mar 2017 21:54:58 -0700 Subject: [PATCH 101/188] HDFS-11545. Propagate DataNode's slow disks info to the NameNode via Heartbeats. Contributed by Hanisha Koneru. --- .../hdfs/server/protocol/SlowDiskReports.java | 126 ++++++++++++++++++ ...atanodeProtocolClientSideTranslatorPB.java | 7 +- ...atanodeProtocolServerSideTranslatorPB.java | 3 +- .../hadoop/hdfs/protocolPB/PBHelper.java | 68 ++++++++++ .../blockmanagement/DatanodeManager.java | 3 +- .../blockmanagement/SlowPeerTracker.java | 2 +- .../hdfs/server/datanode/BPServiceActor.java | 32 +++-- .../datanode/metrics/DataNodeDiskMetrics.java | 34 +++-- .../hdfs/server/namenode/FSNamesystem.java | 6 +- .../server/namenode/NameNodeRpcServer.java | 6 +- .../server/protocol/DatanodeProtocol.java | 3 +- .../src/main/proto/DatanodeProtocol.proto | 15 +++ .../hadoop/hdfs/protocolPB/TestPBHelper.java | 27 ++++ .../TestNameNodePrunesMissingStorages.java | 4 +- .../datanode/InternalDataNodeTestUtils.java | 4 +- .../server/datanode/TestBPOfferService.java | 4 +- .../server/datanode/TestBlockRecovery.java | 4 +- .../datanode/TestBpServiceActorScheduler.java | 20 +-- .../server/datanode/TestDataNodeLifeline.java | 8 +- .../server/datanode/TestDataNodeMXBean.java | 2 +- .../TestDatanodeProtocolRetryPolicy.java | 4 +- .../server/datanode/TestFsDatasetCache.java | 4 +- .../server/datanode/TestStorageReport.java | 4 +- .../namenode/NNThroughputBenchmark.java | 7 +- .../hdfs/server/namenode/NameNodeAdapter.java | 3 +- .../server/namenode/TestDeadDatanode.java | 4 +- 26 files changed, 339 insertions(+), 65 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/SlowDiskReports.java diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/SlowDiskReports.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/SlowDiskReports.java new file mode 100644 index 00000000000..ef4d09e884d --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/SlowDiskReports.java @@ -0,0 +1,126 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.protocol; + +import com.google.common.collect.ImmutableMap; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.Map; + +/** + * A class that allows a DataNode to communicate information about all + * its disks that appear to be slow. + * + * The wire representation of this structure is a list of + * SlowDiskReportProto messages. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public final class SlowDiskReports { + /** + * A map from the DataNode Disk's BasePath to its mean metadata op latency, + * mean read io latency and mean write io latency. + * + * The NameNode must not attempt to interpret the mean latencies + * beyond exposing them as a diagnostic. e.g. metrics. Also, comparing + * latencies across reports from different DataNodes may not be not + * meaningful and must be avoided. + */ + @Nonnull + private final Map> slowDisks; + + /** + * An object representing a SlowPeerReports with no entries. Should + * be used instead of null or creating new objects when there are + * no slow peers to report. + */ + public static final SlowDiskReports EMPTY_REPORT = + new SlowDiskReports(ImmutableMap.of()); + + private SlowDiskReports(Map> slowDisks) { + this.slowDisks = slowDisks; + } + + public static SlowDiskReports create( + @Nullable Map> slowDisks) { + if (slowDisks == null || slowDisks.isEmpty()) { + return EMPTY_REPORT; + } + return new SlowDiskReports(slowDisks); + } + + public Map> getSlowDisks() { + return slowDisks; + } + + public boolean haveSlowDisks() { + return slowDisks.size() > 0; + } + + /** + * Return true if the two objects represent the same set slow disk + * entries. Primarily for unit testing convenience. + */ + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + + if (!(o instanceof SlowDiskReports)) { + return false; + } + + SlowDiskReports that = (SlowDiskReports) o; + + if (this.slowDisks.size() != that.slowDisks.size()) { + return false; + } + + if (!this.slowDisks.keySet().containsAll(that.slowDisks.keySet()) || + !that.slowDisks.keySet().containsAll(this.slowDisks.keySet())) { + return false; + } + + boolean areEqual; + for (String disk : this.slowDisks.keySet()) { + if (!this.slowDisks.get(disk).equals(that.slowDisks.get(disk))) { + return false; + } + } + + return true; + } + + @Override + public int hashCode() { + return slowDisks.hashCode(); + } + + /** + * Lists the types of operations on which disk latencies are measured. + */ + public enum DiskOp { + METADATA, + READ, + WRITE + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java index d9e6026060e..9cc45168835 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java @@ -55,6 +55,7 @@ import org.apache.hadoop.hdfs.server.protocol.HeartbeatResponse; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo.Capability; import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo; +import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports; import org.apache.hadoop.hdfs.server.protocol.SlowPeerReports; import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport; import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks; @@ -136,7 +137,8 @@ public class DatanodeProtocolClientSideTranslatorPB implements int xmitsInProgress, int xceiverCount, int failedVolumes, VolumeFailureSummary volumeFailureSummary, boolean requestFullBlockReportLease, - @Nonnull SlowPeerReports slowPeers) throws IOException { + @Nonnull SlowPeerReports slowPeers, + @Nonnull SlowDiskReports slowDisks) throws IOException { HeartbeatRequestProto.Builder builder = HeartbeatRequestProto.newBuilder() .setRegistration(PBHelper.convert(registration)) .setXmitsInProgress(xmitsInProgress).setXceiverCount(xceiverCount) @@ -156,6 +158,9 @@ public class DatanodeProtocolClientSideTranslatorPB implements if (slowPeers.haveSlowPeers()) { builder.addAllSlowPeers(PBHelper.convertSlowPeerInfo(slowPeers)); } + if (slowDisks.haveSlowDisks()) { + builder.addAllSlowDisks(PBHelper.convertSlowDiskInfo(slowDisks)); + } HeartbeatResponseProto resp; try { resp = rpcProxy.sendHeartbeat(NULL_CONTROLLER, builder.build()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java index b1c8e344fd8..5cba284681f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java @@ -121,7 +121,8 @@ public class DatanodeProtocolServerSideTranslatorPB implements request.getXmitsInProgress(), request.getXceiverCount(), request.getFailedVolumes(), volumeFailureSummary, request.getRequestFullBlockReportLease(), - PBHelper.convertSlowPeerInfo(request.getSlowPeersList())); + PBHelper.convertSlowPeerInfo(request.getSlowPeersList()), + PBHelper.convertSlowDiskInfo(request.getSlowDisksList())); } catch (IOException e) { throw new ServiceException(e); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java index 69c3c8372b9..6539d32b733 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java @@ -47,6 +47,8 @@ import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.FinalizeComm import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.KeyUpdateCommandProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.ReceivedDeletedBlockInfoProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.RegisterCommandProto; +import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos + .SlowDiskReportProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.SlowPeerReportProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.VolumeFailureSummaryProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BlockReportContextProto; @@ -111,6 +113,7 @@ import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo.BlockStat import org.apache.hadoop.hdfs.server.protocol.RegisterCommand; import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog; import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; +import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports; import org.apache.hadoop.hdfs.server.protocol.SlowPeerReports; import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary; @@ -873,6 +876,71 @@ public class PBHelper { return SlowPeerReports.create(slowPeersMap); } + public static List convertSlowDiskInfo( + SlowDiskReports slowDisks) { + if (slowDisks.getSlowDisks().size() == 0) { + return Collections.emptyList(); + } + + List slowDiskInfoProtos = + new ArrayList<>(slowDisks.getSlowDisks().size()); + for (Map.Entry> entry : + slowDisks.getSlowDisks().entrySet()) { + SlowDiskReportProto.Builder builder = SlowDiskReportProto.newBuilder(); + builder.setBasePath(entry.getKey()); + Map value = entry.getValue(); + if (value.get(SlowDiskReports.DiskOp.METADATA) != null) { + builder.setMeanMetadataOpLatency(value.get( + SlowDiskReports.DiskOp.METADATA)); + } + if (value.get(SlowDiskReports.DiskOp.READ) != null) { + builder.setMeanReadIoLatency(value.get( + SlowDiskReports.DiskOp.READ)); + } + if (value.get(SlowDiskReports.DiskOp.WRITE) != null) { + builder.setMeanWriteIoLatency(value.get( + SlowDiskReports.DiskOp.WRITE)); + } + slowDiskInfoProtos.add(builder.build()); + } + + return slowDiskInfoProtos; + } + + public static SlowDiskReports convertSlowDiskInfo( + List slowDiskProtos) { + + // No slow disks, or possibly an older DataNode. + if (slowDiskProtos == null || slowDiskProtos.size() == 0) { + return SlowDiskReports.EMPTY_REPORT; + } + + Map> slowDisksMap = + new HashMap<>(slowDiskProtos.size()); + for (SlowDiskReportProto proto : slowDiskProtos) { + if (!proto.hasBasePath()) { + // The disk basePath should be reported. + continue; + } + Map latencyMap = new HashMap<>(); + if (proto.hasMeanMetadataOpLatency()) { + latencyMap.put(SlowDiskReports.DiskOp.METADATA, + proto.getMeanMetadataOpLatency()); + } + if (proto.hasMeanReadIoLatency()) { + latencyMap.put(SlowDiskReports.DiskOp.READ, + proto.getMeanReadIoLatency()); + } + if (proto.hasMeanWriteIoLatency()) { + latencyMap.put(SlowDiskReports.DiskOp.WRITE, + proto.getMeanWriteIoLatency()); + } + + slowDisksMap.put(proto.getBasePath(), latencyMap); + } + return SlowDiskReports.create(slowDisksMap); + } + public static JournalInfo convert(JournalInfoProto info) { int lv = info.hasLayoutVersion() ? info.getLayoutVersion() : 0; int nsID = info.hasNamespaceID() ? info.getNamespaceID() : 0; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java index 11eb6ac6e51..e22b7afa2a3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java @@ -1584,7 +1584,8 @@ public class DatanodeManager { long cacheCapacity, long cacheUsed, int xceiverCount, int maxTransfers, int failedVolumes, VolumeFailureSummary volumeFailureSummary, - @Nonnull SlowPeerReports slowPeers) throws IOException { + @Nonnull SlowPeerReports slowPeers, + @Nonnull SlowDiskReports slowDisks) throws IOException { final DatanodeDescriptor nodeinfo; try { nodeinfo = getDatanode(nodeReg); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java index c72a6211729..7b24370d7ad 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowPeerTracker.java @@ -58,7 +58,7 @@ public class SlowPeerTracker { /** * Time duration after which a report is considered stale. This is - * set to DFS_DATANODE_SLOW_PEER_REPORT_INTERVAL_KEY * 3 i.e. + * set to DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY * 3 i.e. * maintained for at least two successive reports. */ private final long reportValidityMs; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java index a0ba62716e7..ee5eb72d7eb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java @@ -57,6 +57,7 @@ import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.hdfs.server.protocol.DisallowedDatanodeException; import org.apache.hadoop.hdfs.server.protocol.HeartbeatResponse; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; +import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports; import org.apache.hadoop.hdfs.server.protocol.SlowPeerReports; import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport; import org.apache.hadoop.hdfs.server.protocol.StorageReport; @@ -497,11 +498,15 @@ class BPServiceActor implements Runnable { .getVolumeFailureSummary(); int numFailedVolumes = volumeFailureSummary != null ? volumeFailureSummary.getFailedStorageLocations().length : 0; - final boolean slowPeersReportDue = scheduler.isSlowPeersReportDue(now); + final boolean outliersReportDue = scheduler.isOutliersReportDue(now); final SlowPeerReports slowPeers = - slowPeersReportDue && dn.getPeerMetrics() != null ? + outliersReportDue && dn.getPeerMetrics() != null ? SlowPeerReports.create(dn.getPeerMetrics().getOutliers()) : SlowPeerReports.EMPTY_REPORT; + final SlowDiskReports slowDisks = + outliersReportDue && dn.getDiskMetrics() != null ? + SlowDiskReports.create(dn.getDiskMetrics().getDiskOutliersStats()) : + SlowDiskReports.EMPTY_REPORT; HeartbeatResponse response = bpNamenode.sendHeartbeat(bpRegistration, reports, dn.getFSDataset().getCacheCapacity(), @@ -511,11 +516,12 @@ class BPServiceActor implements Runnable { numFailedVolumes, volumeFailureSummary, requestBlockReportLease, - slowPeers); + slowPeers, + slowDisks); - if (slowPeersReportDue) { + if (outliersReportDue) { // If the report was due and successfully sent, schedule the next one. - scheduler.scheduleNextSlowPeerReport(); + scheduler.scheduleNextOutlierReport(); } return response; } @@ -1095,7 +1101,7 @@ class BPServiceActor implements Runnable { boolean resetBlockReportTime = true; @VisibleForTesting - volatile long nextSlowPeersReportTime = monotonicNow(); + volatile long nextOutliersReportTime = monotonicNow(); private final AtomicBoolean forceFullBlockReport = new AtomicBoolean(false); @@ -1103,14 +1109,14 @@ class BPServiceActor implements Runnable { private final long heartbeatIntervalMs; private final long lifelineIntervalMs; private final long blockReportIntervalMs; - private final long slowPeersReportIntervalMs; + private final long outliersReportIntervalMs; Scheduler(long heartbeatIntervalMs, long lifelineIntervalMs, - long blockReportIntervalMs, long slowPeersReportIntervalMs) { + long blockReportIntervalMs, long outliersReportIntervalMs) { this.heartbeatIntervalMs = heartbeatIntervalMs; this.lifelineIntervalMs = lifelineIntervalMs; this.blockReportIntervalMs = blockReportIntervalMs; - this.slowPeersReportIntervalMs = slowPeersReportIntervalMs; + this.outliersReportIntervalMs = outliersReportIntervalMs; scheduleNextLifeline(nextHeartbeatTime); } @@ -1143,8 +1149,8 @@ class BPServiceActor implements Runnable { lastBlockReportTime = blockReportTime; } - void scheduleNextSlowPeerReport() { - nextSlowPeersReportTime = monotonicNow() + slowPeersReportIntervalMs; + void scheduleNextOutlierReport() { + nextOutliersReportTime = monotonicNow() + outliersReportIntervalMs; } long getLastHearbeatTime() { @@ -1173,8 +1179,8 @@ class BPServiceActor implements Runnable { return nextBlockReportTime - curTime <= 0; } - boolean isSlowPeersReportDue(long curTime) { - return nextSlowPeersReportTime - curTime <= 0; + boolean isOutliersReportDue(long curTime) { + return nextOutliersReportTime - curTime <= 0; } void forceFullBlockReportNow() { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java index 20a356790bf..2602b012a64 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.fsdataset.DataNodeVolumeMetrics; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; +import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports.DiskOp; import org.apache.hadoop.util.Daemon; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,7 +55,7 @@ public class DataNodeDiskMetrics { private volatile boolean shouldRun; private OutlierDetector slowDiskDetector; private Daemon slowDiskDetectionDaemon; - private volatile Map> + private volatile Map> diskOutliersStats = Maps.newHashMap(); public DataNodeDiskMetrics(DataNode dn, long diskOutlierDetectionIntervalMs) { @@ -144,13 +145,13 @@ public class DataNodeDiskMetrics { diskOutliersSet.addAll(writeIoOutliers.keySet()); } - Map> diskStats = + Map> diskStats = Maps.newHashMap(); for (String disk : diskOutliersSet) { - Map diskStat = Maps.newHashMap(); - diskStat.put(DiskOutlierDetectionOp.METADATA, metadataOpStats.get(disk)); - diskStat.put(DiskOutlierDetectionOp.READ, readIoStats.get(disk)); - diskStat.put(DiskOutlierDetectionOp.WRITE, writeIoStats.get(disk)); + Map diskStat = Maps.newHashMap(); + diskStat.put(DiskOp.METADATA, metadataOpStats.get(disk)); + diskStat.put(DiskOp.READ, readIoStats.get(disk)); + diskStat.put(DiskOp.WRITE, writeIoStats.get(disk)); diskStats.put(disk, diskStat); } @@ -158,17 +159,7 @@ public class DataNodeDiskMetrics { LOG.debug("Updated disk outliers."); } - /** - * Lists the types of operations on which disk latencies are measured. - */ - public enum DiskOutlierDetectionOp { - METADATA, - READ, - WRITE - } - - public Map> getDiskOutliersStats() { + public Map> getDiskOutliersStats() { return diskOutliersStats; } @@ -186,7 +177,12 @@ public class DataNodeDiskMetrics { * Use only for testing. */ @VisibleForTesting - public void addSlowDiskForTesting(String slowDiskPath) { - diskOutliersStats.put(slowDiskPath, ImmutableMap.of()); + public void addSlowDiskForTesting(String slowDiskPath, + Map latencies) { + if (latencies == null) { + diskOutliersStats.put(slowDiskPath, ImmutableMap.of()); + } else { + diskOutliersStats.put(slowDiskPath, latencies); + } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 8f09740585a..f405cc0d70a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -88,6 +88,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROU import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY; import static org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.*; +import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports; import static org.apache.hadoop.util.Time.now; import static org.apache.hadoop.util.Time.monotonicNow; import static org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics.TOPMETRICS_METRICS_SOURCE_NAME; @@ -3647,7 +3648,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, int xceiverCount, int xmitsInProgress, int failedVolumes, VolumeFailureSummary volumeFailureSummary, boolean requestFullBlockReportLease, - @Nonnull SlowPeerReports slowPeers) throws IOException { + @Nonnull SlowPeerReports slowPeers, + @Nonnull SlowDiskReports slowDisks) throws IOException { readLock(); try { //get datanode commands @@ -3656,7 +3658,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, DatanodeCommand[] cmds = blockManager.getDatanodeManager().handleHeartbeat( nodeReg, reports, getBlockPoolId(), cacheCapacity, cacheUsed, xceiverCount, maxTransfer, failedVolumes, volumeFailureSummary, - slowPeers); + slowPeers, slowDisks); long blockReportLeaseId = 0; if (requestFullBlockReportLease) { blockReportLeaseId = blockManager.requestBlockReportLeaseId(nodeReg); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index cf14e8afd76..f792e8a9a47 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -155,6 +155,7 @@ import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.hdfs.server.protocol.NodeRegistration; import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; +import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports; import org.apache.hadoop.hdfs.server.protocol.SlowPeerReports; import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport; import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks; @@ -1422,13 +1423,14 @@ public class NameNodeRpcServer implements NamenodeProtocols { int xmitsInProgress, int xceiverCount, int failedVolumes, VolumeFailureSummary volumeFailureSummary, boolean requestFullBlockReportLease, - @Nonnull SlowPeerReports slowPeers) throws IOException { + @Nonnull SlowPeerReports slowPeers, + @Nonnull SlowDiskReports slowDisks) throws IOException { checkNNStartup(); verifyRequest(nodeReg); return namesystem.handleHeartbeat(nodeReg, report, dnCacheCapacity, dnCacheUsed, xceiverCount, xmitsInProgress, failedVolumes, volumeFailureSummary, requestFullBlockReportLease, - slowPeers); + slowPeers, slowDisks); } @Override // DatanodeProtocol diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java index d738e79e495..1f55100af98 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java @@ -122,7 +122,8 @@ public interface DatanodeProtocol { int failedVolumes, VolumeFailureSummary volumeFailureSummary, boolean requestFullBlockReportLease, - @Nonnull SlowPeerReports slowPeers) + @Nonnull SlowPeerReports slowPeers, + @Nonnull SlowDiskReports slowDisks) throws IOException; /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto index 3b25a435812..bf0df5bf144 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto @@ -196,6 +196,7 @@ message VolumeFailureSummaryProto { * cacheUsed - amount of cache used * volumeFailureSummary - info about volume failures * slowPeers - info about peer DataNodes that are suspected to be slow. + * slowDisks - info about DataNode disks that are suspected to be slow. */ message HeartbeatRequestProto { required DatanodeRegistrationProto registration = 1; // Datanode info @@ -208,6 +209,7 @@ message HeartbeatRequestProto { optional VolumeFailureSummaryProto volumeFailureSummary = 8; optional bool requestFullBlockReportLease = 9 [ default = false ]; repeated SlowPeerReportProto slowPeers = 10; + repeated SlowDiskReportProto slowDisks = 11; } /** @@ -405,6 +407,19 @@ message SlowPeerReportProto { optional double aggregateLatency = 2; } +/** + * Information about a single slow disk that may be reported by + * the DataNode to the NameNode as part of the heartbeat request. + * The message includes the disk's basePath, mean metadata op latency, + * mean read io latency and mean write io latency as observed by the DataNode. + */ +message SlowDiskReportProto { + optional string basePath = 1; + optional double meanMetadataOpLatency = 2; + optional double meanReadIoLatency = 3; + optional double meanWriteIoLatency = 4; +} + /** * Protocol used from datanode to the namenode * See the request and response for details of rpc call. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java index ff4b8ec8b1c..314ecf62c4b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.protocolPB; +import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertSame; @@ -792,6 +793,32 @@ public class TestPBHelper { slowPeersConverted2.equals(SlowPeerReports.EMPTY_REPORT)); } + @Test + public void testSlowDiskInfoPBHelper() { + // Test with a map that has a few slow disk entries. + final SlowDiskReports slowDisks = SlowDiskReports.create( + ImmutableMap.of( + "disk1", ImmutableMap.of(SlowDiskReports.DiskOp.METADATA, 0.5), + "disk2", ImmutableMap.of(SlowDiskReports.DiskOp.READ, 1.0, + SlowDiskReports.DiskOp.WRITE, 1.0), + "disk3", ImmutableMap.of(SlowDiskReports.DiskOp.METADATA, 1.2, + SlowDiskReports.DiskOp.READ, 1.5, + SlowDiskReports.DiskOp.WRITE, 1.3))); + SlowDiskReports slowDisksConverted1 = PBHelper.convertSlowDiskInfo( + PBHelper.convertSlowDiskInfo(slowDisks)); + assertTrue( + "Expected map:" + slowDisks + ", got map:" + + slowDisksConverted1.getSlowDisks(), + slowDisksConverted1.equals(slowDisks)); + + // Test with an empty map + SlowDiskReports slowDisksConverted2 = PBHelper.convertSlowDiskInfo( + PBHelper.convertSlowDiskInfo(SlowDiskReports.EMPTY_REPORT)); + assertTrue( + "Expected empty map:" + ", got map:" + slowDisksConverted2, + slowDisksConverted2.equals(SlowDiskReports.EMPTY_REPORT)); + } + private void assertBlockECRecoveryInfoEquals( BlockECReconstructionInfo blkECRecoveryInfo1, BlockECReconstructionInfo blkECRecoveryInfo2) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNameNodePrunesMissingStorages.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNameNodePrunesMissingStorages.java index a5c6e0d1edf..948a8fb2386 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNameNodePrunesMissingStorages.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNameNodePrunesMissingStorages.java @@ -41,6 +41,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi.FsVolumeRef import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; +import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports; import org.apache.hadoop.hdfs.server.protocol.SlowPeerReports; import org.apache.hadoop.hdfs.server.protocol.StorageReport; import org.apache.hadoop.test.GenericTestUtils; @@ -114,7 +115,8 @@ public class TestNameNodePrunesMissingStorages { // Stop the DataNode and send fake heartbeat with missing storage. cluster.stopDataNode(0); cluster.getNameNodeRpc().sendHeartbeat(dnReg, prunedReports, 0L, 0L, 0, 0, - 0, null, true, SlowPeerReports.EMPTY_REPORT); + 0, null, true, SlowPeerReports.EMPTY_REPORT, + SlowDiskReports.EMPTY_REPORT); // Check that the missing storage was pruned. assertThat(dnDescriptor.getStorageInfos().length, is(expectedStoragesAfterTest)); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/InternalDataNodeTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/InternalDataNodeTestUtils.java index cf43fd0fddc..876a8545ec5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/InternalDataNodeTestUtils.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/InternalDataNodeTestUtils.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs.server.datanode; +import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -138,7 +139,8 @@ public class InternalDataNodeTestUtils { Mockito.anyLong(), Mockito.anyInt(), Mockito.anyInt(), Mockito.anyInt(), Mockito.any(VolumeFailureSummary.class), Mockito.anyBoolean(), - Mockito.any(SlowPeerReports.class))).thenReturn( + Mockito.any(SlowPeerReports.class), + Mockito.any(SlowDiskReports.class))).thenReturn( new HeartbeatResponse(new DatanodeCommand[0], new NNHAStatusHeartbeat( HAServiceState.ACTIVE, 1), null, ThreadLocalRandom.current() .nextLong() | 1L)); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java index c6b38eea7db..b9220e0f6c3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs.server.datanode; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY; +import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertNotNull; @@ -154,7 +155,8 @@ public class TestBPOfferService { Mockito.anyInt(), Mockito.any(VolumeFailureSummary.class), Mockito.anyBoolean(), - Mockito.any(SlowPeerReports.class)); + Mockito.any(SlowPeerReports.class), + Mockito.any(SlowDiskReports.class)); mockHaStatuses[nnIdx] = new NNHAStatusHeartbeat(HAServiceState.STANDBY, 0); datanodeCommands[nnIdx] = new DatanodeCommand[0]; return mock; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java index 9f0011c8a5e..e34837c7b72 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs.server.datanode; +import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.Matchers.any; @@ -220,7 +221,8 @@ public class TestBlockRecovery { Mockito.anyInt(), Mockito.any(VolumeFailureSummary.class), Mockito.anyBoolean(), - Mockito.any(SlowPeerReports.class))) + Mockito.any(SlowPeerReports.class), + Mockito.any(SlowDiskReports.class))) .thenReturn(new HeartbeatResponse( new DatanodeCommand[0], new NNHAStatusHeartbeat(HAServiceState.ACTIVE, 1), diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBpServiceActorScheduler.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBpServiceActorScheduler.java index 6435d4d1a5c..753c3a8d6fb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBpServiceActorScheduler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBpServiceActorScheduler.java @@ -51,7 +51,7 @@ public class TestBpServiceActorScheduler { private static final long HEARTBEAT_INTERVAL_MS = 5000; // 5 seconds private static final long LIFELINE_INTERVAL_MS = 3 * HEARTBEAT_INTERVAL_MS; private static final long BLOCK_REPORT_INTERVAL_MS = 10000; // 10 seconds - private static final long SLOW_PEER_REPORT_INTERVAL_MS = 10000; // 10 seconds + private static final long OUTLIER_REPORT_INTERVAL_MS = 10000; // 10 seconds private final Random random = new Random(System.nanoTime()); @Test @@ -182,15 +182,15 @@ public class TestBpServiceActorScheduler { } @Test - public void testSlowPeerReportScheduling() { + public void testOutlierReportScheduling() { for (final long now : getTimestamps()) { Scheduler scheduler = makeMockScheduler(now); - assertTrue(scheduler.isSlowPeersReportDue(now)); - scheduler.scheduleNextSlowPeerReport(); - assertFalse(scheduler.isSlowPeersReportDue(now)); - assertFalse(scheduler.isSlowPeersReportDue(now + 1)); - assertTrue(scheduler.isSlowPeersReportDue( - now + SLOW_PEER_REPORT_INTERVAL_MS)); + assertTrue(scheduler.isOutliersReportDue(now)); + scheduler.scheduleNextOutlierReport(); + assertFalse(scheduler.isOutliersReportDue(now)); + assertFalse(scheduler.isOutliersReportDue(now + 1)); + assertTrue(scheduler.isOutliersReportDue( + now + OUTLIER_REPORT_INTERVAL_MS)); } } @@ -198,11 +198,11 @@ public class TestBpServiceActorScheduler { LOG.info("Using now = " + now); Scheduler mockScheduler = spy(new Scheduler( HEARTBEAT_INTERVAL_MS, LIFELINE_INTERVAL_MS, - BLOCK_REPORT_INTERVAL_MS, SLOW_PEER_REPORT_INTERVAL_MS)); + BLOCK_REPORT_INTERVAL_MS, OUTLIER_REPORT_INTERVAL_MS)); doReturn(now).when(mockScheduler).monotonicNow(); mockScheduler.nextBlockReportTime = now; mockScheduler.nextHeartbeatTime = now; - mockScheduler.nextSlowPeersReportTime = now; + mockScheduler.nextOutliersReportTime = now; return mockScheduler; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeLifeline.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeLifeline.java index 8a9f0b8da1a..28427bce75a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeLifeline.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeLifeline.java @@ -23,6 +23,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LIFELINE_RPC_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_INTERVAL_KEY; + +import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports; import static org.apache.hadoop.test.MetricsAsserts.getLongCounter; import static org.apache.hadoop.test.MetricsAsserts.getMetrics; import static org.junit.Assert.assertEquals; @@ -169,7 +171,8 @@ public class TestDataNodeLifeline { anyInt(), any(VolumeFailureSummary.class), anyBoolean(), - any(SlowPeerReports.class)); + any(SlowPeerReports.class), + any(SlowDiskReports.class)); // Intercept lifeline to trigger latch count-down on each call. doAnswer(new LatchCountingAnswer(lifelinesSent)) @@ -233,7 +236,8 @@ public class TestDataNodeLifeline { anyInt(), any(VolumeFailureSummary.class), anyBoolean(), - any(SlowPeerReports.class)); + any(SlowPeerReports.class), + any(SlowDiskReports.class)); // While waiting on the latch for the expected number of heartbeat messages, // poll DataNode tracking information. We expect that the DataNode always diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMXBean.java index b6277d60a8f..b80976a2c3f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMXBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMXBean.java @@ -227,7 +227,7 @@ public class TestDataNodeMXBean { Assert.assertEquals(datanodes.size(), 1); DataNode datanode = datanodes.get(0); String slowDiskPath = "test/data1/slowVolume"; - datanode.getDiskMetrics().addSlowDiskForTesting(slowDiskPath); + datanode.getDiskMetrics().addSlowDiskForTesting(slowDiskPath, null); MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); ObjectName mxbeanName = new ObjectName( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDatanodeProtocolRetryPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDatanodeProtocolRetryPolicy.java index c94f74ecc52..bb1d9eff0ee 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDatanodeProtocolRetryPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDatanodeProtocolRetryPolicy.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs.server.datanode; +import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -220,7 +221,8 @@ public class TestDatanodeProtocolRetryPolicy { Mockito.anyInt(), Mockito.any(VolumeFailureSummary.class), Mockito.anyBoolean(), - Mockito.any(SlowPeerReports.class)); + Mockito.any(SlowPeerReports.class), + Mockito.any(SlowDiskReports.class)); dn = new DataNode(conf, locations, null, null) { @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java index eb015c03573..28bf13bb395 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.server.datanode; +import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports; import static org.apache.hadoop.test.MetricsAsserts.getMetrics; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -173,7 +174,8 @@ public class TestFsDatasetCache { (DatanodeRegistration) any(), (StorageReport[]) any(), anyLong(), anyLong(), anyInt(), anyInt(), anyInt(), (VolumeFailureSummary) any(), - anyBoolean(), any(SlowPeerReports.class)); + anyBoolean(), any(SlowPeerReports.class), + any(SlowDiskReports.class)); } private static DatanodeCommand[] cacheBlock(HdfsBlockLocation loc) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestStorageReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestStorageReport.java index 2b793e9caaf..5f62ddb084b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestStorageReport.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestStorageReport.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; +import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports; import org.apache.hadoop.hdfs.server.protocol.SlowPeerReports; import org.apache.hadoop.hdfs.server.protocol.StorageReport; import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary; @@ -108,7 +109,8 @@ public class TestStorageReport { captor.capture(), anyLong(), anyLong(), anyInt(), anyInt(), anyInt(), Mockito.any(VolumeFailureSummary.class), Mockito.anyBoolean(), - Mockito.any(SlowPeerReports.class)); + Mockito.any(SlowPeerReports.class), + Mockito.any(SlowDiskReports.class)); StorageReport[] reports = captor.getValue(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java index b86b3fb515e..c1f0a7be3f0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java @@ -64,6 +64,7 @@ import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo; +import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports; import org.apache.hadoop.hdfs.server.protocol.SlowPeerReports; import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport; import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks; @@ -953,7 +954,8 @@ public class NNThroughputBenchmark implements Tool { DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED, 0L) }; DatanodeCommand[] cmds = dataNodeProto.sendHeartbeat(dnRegistration, rep, 0L, 0L, 0, 0, 0, null, true, - SlowPeerReports.EMPTY_REPORT).getCommands(); + SlowPeerReports.EMPTY_REPORT, SlowDiskReports.EMPTY_REPORT) + .getCommands(); if(cmds != null) { for (DatanodeCommand cmd : cmds ) { if(LOG.isDebugEnabled()) { @@ -1003,7 +1005,8 @@ public class NNThroughputBenchmark implements Tool { false, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED, 0) }; DatanodeCommand[] cmds = dataNodeProto.sendHeartbeat(dnRegistration, rep, 0L, 0L, 0, 0, 0, null, true, - SlowPeerReports.EMPTY_REPORT).getCommands(); + SlowPeerReports.EMPTY_REPORT, SlowDiskReports.EMPTY_REPORT) + .getCommands(); if (cmds != null) { for (DatanodeCommand cmd : cmds) { if (cmd.getAction() == DatanodeProtocol.DNA_TRANSFER) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java index 2b8faf46a98..33af59e0c3d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.server.namenode; +import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports; import static org.mockito.Mockito.spy; import java.io.File; @@ -124,7 +125,7 @@ public class NameNodeAdapter { return namesystem.handleHeartbeat(nodeReg, BlockManagerTestUtil.getStorageReportsForDatanode(dd), dd.getCacheCapacity(), dd.getCacheRemaining(), 0, 0, 0, null, true, - SlowPeerReports.EMPTY_REPORT); + SlowPeerReports.EMPTY_REPORT, SlowDiskReports.EMPTY_REPORT); } public static boolean setReplication(final FSNamesystem ns, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java index b9161c3438a..6df8fcfb5d6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.server.namenode; +import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.fail; @@ -134,7 +135,8 @@ public class TestDeadDatanode { false, 0, 0, 0, 0, 0) }; DatanodeCommand[] cmd = dnp.sendHeartbeat(reg, rep, 0L, 0L, 0, 0, 0, null, true, - SlowPeerReports.EMPTY_REPORT).getCommands(); + SlowPeerReports.EMPTY_REPORT, SlowDiskReports.EMPTY_REPORT) + .getCommands(); assertEquals(1, cmd.length); assertEquals(cmd[0].getAction(), RegisterCommand.REGISTER .getAction()); From 1a98a279f45c0ce018a9c7f176d26098aa74b132 Mon Sep 17 00:00:00 2001 From: Sunil G Date: Tue, 21 Mar 2017 13:46:31 +0530 Subject: [PATCH 102/188] YARN-6362. Use frontend-maven-plugin 0.0.22 version for new yarn ui. Contributed by Kai Sasaki. --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml index 31d2f5e6880..40eb54d9968 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/pom.xml @@ -32,8 +32,8 @@ pom ${basedir}/target/src/main/webapp ${basedir}/target/src/main/webapp/node/node - v4.4.5 - 2.15.5 + v5.7.1 + 3.6.0 false @@ -123,6 +123,7 @@ com.github.eirslett frontend-maven-plugin + 0.0.22 ${webappTgtDir} From 3b908f71c5825a8fd6ded2a6108eb4c6c4a5b9c4 Mon Sep 17 00:00:00 2001 From: Yiqun Lin Date: Tue, 21 Mar 2017 17:44:17 +0800 Subject: [PATCH 103/188] HDFS-11358. DiskBalancer: Report command supports reading nodes from host file. Contributed by Yiqun Lin. --- .../diskbalancer/DiskBalancerException.java | 1 + .../server/diskbalancer/command/Command.java | 29 ++++++-- .../diskbalancer/command/ReportCommand.java | 6 +- .../src/site/markdown/HDFSCommands.md | 2 +- .../src/site/markdown/HDFSDiskbalancer.md | 4 +- .../command/TestDiskBalancerCommand.java | 71 +++++++++++++++++-- 6 files changed, 95 insertions(+), 18 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerException.java index a730a57924f..642cf2186d4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerException.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/DiskBalancerException.java @@ -41,6 +41,7 @@ public class DiskBalancerException extends IOException { UNKNOWN_KEY, INVALID_NODE, DATANODE_STATUS_NOT_REGULAR, + INVALID_HOST_FILE_PATH, } private final Result result; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java index 22fd5beca7e..eeb7241cfbb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java @@ -49,17 +49,17 @@ import org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerVolumeSe import org.apache.hadoop.hdfs.tools.DiskBalancerCLI; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.HostsFileReader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.Closeable; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.PrintStream; import java.net.InetSocketAddress; import java.net.URI; import java.net.URL; -import java.nio.charset.Charset; -import java.nio.file.Files; import java.nio.file.Paths; import java.text.SimpleDateFormat; import java.util.Collections; @@ -268,16 +268,33 @@ public abstract class Command extends Configured implements Closeable { if ((listArg == null) || listArg.isEmpty()) { return resultSet; } + if (listArg.startsWith("file://")) { listURL = new URL(listArg); - byte[] data = Files.readAllBytes(Paths.get(listURL.getPath())); - nodeData = new String(data, Charset.forName("UTF-8")); + try { + HostsFileReader.readFileToSet("include", + Paths.get(listURL.getPath()).toString(), resultSet); + } catch (FileNotFoundException e) { + String warnMsg = String + .format("The input host file path '%s' is not a valid path. " + + "Please make sure the host file exists.", listArg); + throw new DiskBalancerException(warnMsg, + DiskBalancerException.Result.INVALID_HOST_FILE_PATH); + } } else { nodeData = listArg; + String[] nodes = nodeData.split(","); + + if (nodes.length == 0) { + String warnMsg = "The number of input nodes is 0. " + + "Please input the valid nodes."; + throw new DiskBalancerException(warnMsg, + DiskBalancerException.Result.INVALID_NODE); + } + + Collections.addAll(resultSet, nodes); } - String[] nodes = nodeData.split(","); - Collections.addAll(resultSet, nodes); return resultSet; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java index e10ffacc735..b224b11d47c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java @@ -148,7 +148,9 @@ public class ReportCommand extends Command { * Reporting volume information for specific DataNode(s) */ outputLine = String.format( - "Reporting volume information for DataNode(s) '%s'.", nodeVal); + "Reporting volume information for DataNode(s). " + + "These DataNode(s) are parsed from '%s'.", nodeVal); + recordOutput(result, outputLine); List dbdns = Lists.newArrayList(); @@ -224,7 +226,7 @@ public class ReportCommand extends Command { + "hdfs diskbalancer -report\n" + "hdfs diskbalancer -report -top 5\n" + "hdfs diskbalancer -report " - + "-node [,...]"; + + "-node | [,...]"; HelpFormatter helpFormatter = new HelpFormatter(); helpFormatter.printHelp("hdfs diskbalancer -fs http://namenode.uri " + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md index d031dad1236..1ee700316ea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md @@ -418,7 +418,7 @@ Usage: [-query ] [-cancel ] [-cancel -node ] - [-report -node [,...]] + [-report -node | [,...]] [-report -node -top ] | COMMAND\_OPTION | Description | diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md index b6eb62e7004..6e1bd41510f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md @@ -102,9 +102,9 @@ or Plan ID can be read from datanode using query command. ### Report -Report command provides detailed report of specified node(s) or top nodes that will benefit from running disk balancer. +Report command provides detailed report of specified node(s) or top nodes that will benefit from running disk balancer. The node(s) can be specified by a host file or comma-separated list of nodes. -`hdfs diskbalancer -fs http://namenode.uri -report -node [,...]` +`hdfs diskbalancer -fs http://namenode.uri -report -node | [,...]` or diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java index ad16cfa7a6b..b0b0b0c563c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java @@ -34,6 +34,8 @@ import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileWriter; import java.io.PrintStream; import java.net.URI; import java.util.List; @@ -675,14 +677,18 @@ public class TestDiskBalancerCommand { REPORT, NODE, dataNodeUuid1, dataNodeUuid2); final String cmdLine = String.format("hdfs diskbalancer %s", planArg); List outputs = runCommand(cmdLine, cluster); + verifyOutputsOfReportCommand(outputs, dataNodeUuid1, dataNodeUuid2, true); + } + + private void verifyOutputsOfReportCommand(List outputs, + String dataNodeUuid1, String dataNodeUuid2, boolean inputNodesStr) { + assertThat(outputs.get(0), containsString("Processing report command")); + if (inputNodesStr) { + assertThat(outputs.get(1), + is(allOf(containsString("Reporting volume information for DataNode"), + containsString(dataNodeUuid1), containsString(dataNodeUuid2)))); + } - assertThat( - outputs.get(0), - containsString("Processing report command")); - assertThat( - outputs.get(1), - is(allOf(containsString("Reporting volume information for DataNode"), - containsString(dataNodeUuid1), containsString(dataNodeUuid2)))); // Since the order of input nodes will be disrupted when parse // the node string, we should compare UUID with both output lines. assertTrue(outputs.get(2).contains(dataNodeUuid1) @@ -714,4 +720,55 @@ public class TestDiskBalancerCommand { , invalidNode, invalidNode); assertTrue(outputs.get(2).contains(invalidNodeInfo)); } + + @Test(timeout = 60000) + public void testReportCommandWithNullNodes() throws Exception { + // don't input nodes + final String planArg = String.format("-%s -%s ,", REPORT, NODE); + final String cmdLine = String.format("hdfs diskbalancer %s", planArg); + List outputs = runCommand(cmdLine, cluster); + + String invalidNodeInfo = "The number of input nodes is 0. " + + "Please input the valid nodes."; + assertTrue(outputs.get(2).contains(invalidNodeInfo)); + } + + @Test(timeout = 60000) + public void testReportCommandWithReadingHostFile() throws Exception { + final String testDir = GenericTestUtils.getTestDir().getAbsolutePath(); + File includeFile = new File(testDir, "diskbalancer.include"); + String filePath = testDir + "/diskbalancer.include"; + + String dataNodeUuid1 = cluster.getDataNodes().get(0).getDatanodeUuid(); + String dataNodeUuid2 = cluster.getDataNodes().get(1).getDatanodeUuid(); + + FileWriter fw = new FileWriter(filePath); + fw.write("#This-is-comment\n"); + fw.write(dataNodeUuid1 + "\n"); + fw.write(dataNodeUuid2 + "\n"); + fw.close(); + + final String planArg = String.format("-%s -%s file://%s", + REPORT, NODE, filePath); + final String cmdLine = String.format("hdfs diskbalancer %s", planArg); + List outputs = runCommand(cmdLine, cluster); + + verifyOutputsOfReportCommand(outputs, dataNodeUuid1, dataNodeUuid2, false); + includeFile.delete(); + } + + @Test(timeout = 60000) + public void testReportCommandWithInvalidHostFilePath() throws Exception { + final String testDir = GenericTestUtils.getTestDir().getAbsolutePath(); + String invalidFilePath = testDir + "/diskbalancer-invalid.include"; + + final String planArg = String.format("-%s -%s file://%s", + REPORT, NODE, invalidFilePath); + final String cmdLine = String.format("hdfs diskbalancer %s", planArg); + List outputs = runCommand(cmdLine, cluster); + + String invalidNodeInfo = String.format( + "The input host file path 'file://%s' is not a valid path.", invalidFilePath); + assertTrue(outputs.get(2).contains(invalidNodeInfo)); + } } From 2841666f1f2dec96761a0aa34a69cbb20297aa14 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Tue, 21 Mar 2017 13:15:15 +0000 Subject: [PATCH 104/188] HADOOP-14204 S3A multipart commit failing, "UnsupportedOperationException at java.util.Collections$UnmodifiableList.sort". Contributed by Steve Loughran (cherry picked from commit 595501c9ddacd5fdb740ac1bf9bfeb0ffbbd6c0e) --- .../src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index e0ff5a884ba..6c8941253e4 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -2403,11 +2403,13 @@ public class S3AFileSystem extends FileSystem { "No partitions have been uploaded"); LOG.debug("Completing multipart upload {} with {} parts", uploadId, partETags.size()); + // a copy of the list is required, so that the AWS SDK doesn't + // attempt to sort an unmodifiable list. return s3.completeMultipartUpload( new CompleteMultipartUploadRequest(bucket, key, uploadId, - partETags)); + new ArrayList<>(partETags))); } /** From 74af0bdf68d56f02e92dad2411b26d1cde3dc703 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Wed, 22 Mar 2017 02:45:32 +0900 Subject: [PATCH 105/188] HADOOP-14187. Update ZooKeeper dependency to 3.4.9 and Curator dependency to 2.12.0. Contributed by Tsuyoshi Ozawa. --- hadoop-project/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index fbe961a075b..8de7703359a 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -81,8 +81,8 @@ 2.5.0 ${env.HADOOP_PROTOC_PATH} - 3.4.6 - 2.7.1 + 3.4.9 + 2.12.0 3.0.0 6.0.48 From c01d15ab2731b6710c94ff3bfa37d496a87b0c9f Mon Sep 17 00:00:00 2001 From: Junping Du Date: Tue, 21 Mar 2017 00:28:05 -0700 Subject: [PATCH 106/188] YARN-6367. YARN logs CLI needs alway check containerLogsInfo/containerLogInfo before parse the JSON object from NMWebService. Contributed by Xuan Gong. --- .../java/org/apache/hadoop/yarn/client/cli/LogsCLI.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java index 8407b1930d1..4125a81f005 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java @@ -430,6 +430,9 @@ public class LogsCLI extends Configured implements Tool { try { JSONArray array = new JSONArray(); JSONObject json = response.getEntity(JSONObject.class); + if (!json.has("containerLogsInfo")) { + return logFileInfos; + } Object logsInfoObj = json.get("containerLogsInfo"); if (logsInfoObj instanceof JSONObject) { array.put((JSONObject)logsInfoObj); @@ -443,6 +446,9 @@ public class LogsCLI extends Configured implements Tool { JSONObject log = array.getJSONObject(i); String aggregateType = log.has("logAggregationType") ? log.getString("logAggregationType") : "N/A"; + if (!log.has("containerLogInfo")) { + continue; + } Object ob = log.get("containerLogInfo"); if (ob instanceof JSONArray) { JSONArray obArray = (JSONArray)ob; From 0a05c5c5989edeba2cffe16e80350245778cefce Mon Sep 17 00:00:00 2001 From: Daniel Templeton Date: Tue, 21 Mar 2017 14:15:40 -0700 Subject: [PATCH 107/188] YARN-6284. hasAlreadyRun should be final in ResourceManager.StandByTransitionRunnable (Contributed by Laura Adams via Daniel Templeton) --- .../hadoop/yarn/server/resourcemanager/ResourceManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 58e4077feb9..b62315e7e69 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -846,7 +846,7 @@ public class ResourceManager extends CompositeService implements Recoverable { private class StandByTransitionRunnable implements Runnable { // The atomic variable to make sure multiple threads with the same runnable // run only once. - private AtomicBoolean hasAlreadyRun = new AtomicBoolean(false); + private final AtomicBoolean hasAlreadyRun = new AtomicBoolean(false); @Override public void run() { From cc938e99ec0904824c8072184eff75619fcaf040 Mon Sep 17 00:00:00 2001 From: Robert Kanter Date: Tue, 21 Mar 2017 15:21:11 -0700 Subject: [PATCH 108/188] YARN-6326. Shouldn't use AppAttemptIds to fetch applications while AM Simulator tracks app in SLS (yufeigu via rkanter) --- .../yarn/sls/appmaster/AMSimulator.java | 9 +- .../sls/scheduler/FairSchedulerMetrics.java | 314 ++++++++---------- .../scheduler/ResourceSchedulerWrapper.java | 11 +- .../sls/scheduler/SLSCapacityScheduler.java | 9 +- .../yarn/sls/scheduler/SchedulerMetrics.java | 71 ++-- .../yarn/sls/scheduler/SchedulerWrapper.java | 25 +- .../yarn/sls/appmaster/TestAMSimulator.java | 56 +++- 7 files changed, 253 insertions(+), 242 deletions(-) diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java index 5b03d514257..0573bae0ecf 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java @@ -62,10 +62,6 @@ import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.log4j.Logger; @@ -335,13 +331,12 @@ public abstract class AMSimulator extends TaskRunner.Task { private void trackApp() { if (isTracked) { ((SchedulerWrapper) rm.getResourceScheduler()) - .addTrackedApp(appAttemptId, oldAppId); + .addTrackedApp(appId, oldAppId); } } public void untrackApp() { if (isTracked) { - ((SchedulerWrapper) rm.getResourceScheduler()) - .removeTrackedApp(appAttemptId, oldAppId); + ((SchedulerWrapper) rm.getResourceScheduler()).removeTrackedApp(oldAppId); } } diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java index 3b539fa6be4..08362b1c609 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java @@ -18,16 +18,17 @@ package org.apache.hadoop.yarn.sls.scheduler; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair - .FSAppAttempt; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.Schedulable; +import org.apache.hadoop.yarn.sls.SLSRunner; import com.codahale.metrics.Gauge; -import org.apache.hadoop.yarn.sls.SLSRunner; @Private @Unstable @@ -37,114 +38,131 @@ public class FairSchedulerMetrics extends SchedulerMetrics { private int totalVCores = Integer.MAX_VALUE; private boolean maxReset = false; + @VisibleForTesting + public enum Metric { + DEMAND("demand"), + USAGE("usage"), + MINSHARE("minshare"), + MAXSHARE("maxshare"), + FAIRSHARE("fairshare"); + + private String value; + + Metric(String value) { + this.value = value; + } + + @VisibleForTesting + public String getValue() { + return value; + } + } + public FairSchedulerMetrics() { super(); - appTrackedMetrics.add("demand.memory"); - appTrackedMetrics.add("demand.vcores"); - appTrackedMetrics.add("usage.memory"); - appTrackedMetrics.add("usage.vcores"); - appTrackedMetrics.add("minshare.memory"); - appTrackedMetrics.add("minshare.vcores"); - appTrackedMetrics.add("maxshare.memory"); - appTrackedMetrics.add("maxshare.vcores"); - appTrackedMetrics.add("fairshare.memory"); - appTrackedMetrics.add("fairshare.vcores"); - queueTrackedMetrics.add("demand.memory"); - queueTrackedMetrics.add("demand.vcores"); - queueTrackedMetrics.add("usage.memory"); - queueTrackedMetrics.add("usage.vcores"); - queueTrackedMetrics.add("minshare.memory"); - queueTrackedMetrics.add("minshare.vcores"); - queueTrackedMetrics.add("maxshare.memory"); - queueTrackedMetrics.add("maxshare.vcores"); - queueTrackedMetrics.add("fairshare.memory"); - queueTrackedMetrics.add("fairshare.vcores"); + + for (Metric metric: Metric.values()) { + appTrackedMetrics.add(metric.value + ".memory"); + appTrackedMetrics.add(metric.value + ".vcores"); + queueTrackedMetrics.add(metric.value + ".memory"); + queueTrackedMetrics.add(metric.value + ".vcores"); + } } - + + private long getMemorySize(Schedulable schedulable, Metric metric) { + if (schedulable != null) { + switch (metric) { + case DEMAND: + return schedulable.getDemand().getMemorySize(); + case USAGE: + return schedulable.getResourceUsage().getMemorySize(); + case MINSHARE: + return schedulable.getMinShare().getMemorySize(); + case MAXSHARE: + return schedulable.getMaxShare().getMemorySize(); + case FAIRSHARE: + return schedulable.getFairShare().getMemorySize(); + default: + return 0L; + } + } + + return 0L; + } + + private int getVirtualCores(Schedulable schedulable, Metric metric) { + if (schedulable != null) { + switch (metric) { + case DEMAND: + return schedulable.getDemand().getVirtualCores(); + case USAGE: + return schedulable.getResourceUsage().getVirtualCores(); + case MINSHARE: + return schedulable.getMinShare().getVirtualCores(); + case MAXSHARE: + return schedulable.getMaxShare().getVirtualCores(); + case FAIRSHARE: + return schedulable.getFairShare().getVirtualCores(); + default: + return 0; + } + } + + return 0; + } + + private void registerAppMetrics(ApplicationId appId, String oldAppId, + Metric metric) { + metrics.register( + "variable.app." + oldAppId + "." + metric.value + ".memory", + new Gauge() { + @Override + public Long getValue() { + return getMemorySize((FSAppAttempt)getSchedulerAppAttempt(appId), + metric); + } + } + ); + + metrics.register( + "variable.app." + oldAppId + "." + metric.value + ".vcores", + new Gauge() { + @Override + public Integer getValue() { + return getVirtualCores((FSAppAttempt)getSchedulerAppAttempt(appId), + metric); + } + } + ); + } + @Override - public void trackApp(ApplicationAttemptId appAttemptId, String oldAppId) { - super.trackApp(appAttemptId, oldAppId); - FairScheduler fair = (FairScheduler) scheduler; - final FSAppAttempt app = fair.getSchedulerApp(appAttemptId); - metrics.register("variable.app." + oldAppId + ".demand.memory", - new Gauge() { - @Override - public Long getValue() { - return app.getDemand().getMemorySize(); + public void trackApp(ApplicationId appId, String oldAppId) { + super.trackApp(appId, oldAppId); + + for (Metric metric: Metric.values()) { + registerAppMetrics(appId, oldAppId, metric); + } + } + + private void registerQueueMetrics(FSQueue queue, Metric metric) { + metrics.register( + "variable.queue." + queue.getName() + "." + metric.value + ".memory", + new Gauge() { + @Override + public Long getValue() { + return getMemorySize(queue, metric); + } } - } ); - metrics.register("variable.app." + oldAppId + ".demand.vcores", - new Gauge() { - @Override - public Integer getValue() { - return app.getDemand().getVirtualCores(); + metrics.register( + "variable.queue." + queue.getName() + "." + metric.value + ".vcores", + new Gauge() { + @Override + public Integer getValue() { + return getVirtualCores(queue, metric); + } } - } - ); - metrics.register("variable.app." + oldAppId + ".usage.memory", - new Gauge() { - @Override - public Long getValue() { - return app.getResourceUsage().getMemorySize(); - } - } - ); - metrics.register("variable.app." + oldAppId + ".usage.vcores", - new Gauge() { - @Override - public Integer getValue() { - return app.getResourceUsage().getVirtualCores(); - } - } - ); - metrics.register("variable.app." + oldAppId + ".minshare.memory", - new Gauge() { - @Override - public Long getValue() { - return app.getMinShare().getMemorySize(); - } - } - ); - metrics.register("variable.app." + oldAppId + ".minshare.vcores", - new Gauge() { - @Override - public Long getValue() { - return app.getMinShare().getMemorySize(); - } - } - ); - metrics.register("variable.app." + oldAppId + ".maxshare.memory", - new Gauge() { - @Override - public Long getValue() { - return Math.min(app.getMaxShare().getMemorySize(), totalMemoryMB); - } - } - ); - metrics.register("variable.app." + oldAppId + ".maxshare.vcores", - new Gauge() { - @Override - public Integer getValue() { - return Math.min(app.getMaxShare().getVirtualCores(), totalVCores); - } - } - ); - metrics.register("variable.app." + oldAppId + ".fairshare.memory", - new Gauge() { - @Override - public Integer getValue() { - return app.getFairShare().getVirtualCores(); - } - } - ); - metrics.register("variable.app." + oldAppId + ".fairshare.vcores", - new Gauge() { - @Override - public Integer getValue() { - return app.getFairShare().getVirtualCores(); - } - } ); } @@ -153,54 +171,11 @@ public class FairSchedulerMetrics extends SchedulerMetrics { trackedQueues.add(queueName); FairScheduler fair = (FairScheduler) scheduler; final FSQueue queue = fair.getQueueManager().getQueue(queueName); - metrics.register("variable.queue." + queueName + ".demand.memory", - new Gauge() { - @Override - public Long getValue() { - return queue.getDemand().getMemorySize(); - } - } - ); - metrics.register("variable.queue." + queueName + ".demand.vcores", - new Gauge() { - @Override - public Integer getValue() { - return queue.getDemand().getVirtualCores(); - } - } - ); - metrics.register("variable.queue." + queueName + ".usage.memory", - new Gauge() { - @Override - public Long getValue() { - return queue.getResourceUsage().getMemorySize(); - } - } - ); - metrics.register("variable.queue." + queueName + ".usage.vcores", - new Gauge() { - @Override - public Integer getValue() { - return queue.getResourceUsage().getVirtualCores(); - } - } - ); - metrics.register("variable.queue." + queueName + ".minshare.memory", - new Gauge() { - @Override - public Long getValue() { - return queue.getMinShare().getMemorySize(); - } - } - ); - metrics.register("variable.queue." + queueName + ".minshare.vcores", - new Gauge() { - @Override - public Integer getValue() { - return queue.getMinShare().getVirtualCores(); - } - } - ); + registerQueueMetrics(queue, Metric.DEMAND); + registerQueueMetrics(queue, Metric.USAGE); + registerQueueMetrics(queue, Metric.MINSHARE); + registerQueueMetrics(queue, Metric.FAIRSHARE); + metrics.register("variable.queue." + queueName + ".maxshare.memory", new Gauge() { @Override @@ -233,36 +208,17 @@ public class FairSchedulerMetrics extends SchedulerMetrics { } } ); - metrics.register("variable.queue." + queueName + ".fairshare.memory", - new Gauge() { - @Override - public Long getValue() { - return queue.getFairShare().getMemorySize(); - } - } - ); - metrics.register("variable.queue." + queueName + ".fairshare.vcores", - new Gauge() { - @Override - public Integer getValue() { - return queue.getFairShare().getVirtualCores(); - } - } - ); } @Override public void untrackQueue(String queueName) { trackedQueues.remove(queueName); - metrics.remove("variable.queue." + queueName + ".demand.memory"); - metrics.remove("variable.queue." + queueName + ".demand.vcores"); - metrics.remove("variable.queue." + queueName + ".usage.memory"); - metrics.remove("variable.queue." + queueName + ".usage.vcores"); - metrics.remove("variable.queue." + queueName + ".minshare.memory"); - metrics.remove("variable.queue." + queueName + ".minshare.vcores"); - metrics.remove("variable.queue." + queueName + ".maxshare.memory"); - metrics.remove("variable.queue." + queueName + ".maxshare.vcores"); - metrics.remove("variable.queue." + queueName + ".fairshare.memory"); - metrics.remove("variable.queue." + queueName + ".fairshare.vcores"); + + for (Metric metric: Metric.values()) { + metrics.remove("variable.queue." + queueName + "." + + metric.value + ".memory"); + metrics.remove("variable.queue." + queueName + "." + + metric.value + ".vcores"); + } } } diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java index df8323a081b..a4b8e642ba5 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java @@ -70,7 +70,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedContainerChangeRequest; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; @@ -793,17 +792,15 @@ final public class ResourceSchedulerWrapper } // API open to out classes - public void addTrackedApp(ApplicationAttemptId appAttemptId, - String oldAppId) { + public void addTrackedApp(ApplicationId appId, String oldAppId) { if (metricsON) { - schedulerMetrics.trackApp(appAttemptId, oldAppId); + schedulerMetrics.trackApp(appId, oldAppId); } } - public void removeTrackedApp(ApplicationAttemptId appAttemptId, - String oldAppId) { + public void removeTrackedApp(String oldAppId) { if (metricsON) { - schedulerMetrics.untrackApp(appAttemptId, oldAppId); + schedulerMetrics.untrackApp(oldAppId); } } diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java index cd4377e1eee..6ea2ab0f1e5 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java @@ -839,17 +839,16 @@ public class SLSCapacityScheduler extends CapacityScheduler implements } // API open to out classes - public void addTrackedApp(ApplicationAttemptId appAttemptId, + public void addTrackedApp(ApplicationId appId, String oldAppId) { if (metricsON) { - schedulerMetrics.trackApp(appAttemptId, oldAppId); + schedulerMetrics.trackApp(appId, oldAppId); } } - public void removeTrackedApp(ApplicationAttemptId appAttemptId, - String oldAppId) { + public void removeTrackedApp(String oldAppId) { if (metricsON) { - schedulerMetrics.untrackApp(appAttemptId, oldAppId); + schedulerMetrics.untrackApp(oldAppId); } } diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java index ecf516d7c98..8645a697c1d 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java @@ -23,11 +23,11 @@ import java.util.Set; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler - .ResourceScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler - .SchedulerAppReport; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; import com.codahale.metrics.Gauge; import com.codahale.metrics.MetricRegistry; @@ -42,42 +42,61 @@ public abstract class SchedulerMetrics { protected Set queueTrackedMetrics; public SchedulerMetrics() { - appTrackedMetrics = new HashSet(); + appTrackedMetrics = new HashSet<>(); appTrackedMetrics.add("live.containers"); appTrackedMetrics.add("reserved.containers"); - queueTrackedMetrics = new HashSet(); + queueTrackedMetrics = new HashSet<>(); } public void init(ResourceScheduler scheduler, MetricRegistry metrics) { this.scheduler = scheduler; - this.trackedQueues = new HashSet(); + this.trackedQueues = new HashSet<>(); this.metrics = metrics; } - - public void trackApp(final ApplicationAttemptId appAttemptId, - String oldAppId) { + + protected SchedulerApplicationAttempt getSchedulerAppAttempt( + ApplicationId appId) { + AbstractYarnScheduler yarnScheduler = (AbstractYarnScheduler)scheduler; + SchedulerApplication app = (SchedulerApplication)yarnScheduler + .getSchedulerApplications().get(appId); + if (app == null) { + return null; + } + return app.getCurrentAppAttempt(); + } + + public void trackApp(final ApplicationId appId, String oldAppId) { metrics.register("variable.app." + oldAppId + ".live.containers", - new Gauge() { - @Override - public Integer getValue() { - SchedulerAppReport app = scheduler.getSchedulerAppInfo(appAttemptId); - return app.getLiveContainers().size(); + new Gauge() { + @Override + public Integer getValue() { + SchedulerApplicationAttempt appAttempt = + getSchedulerAppAttempt(appId); + if (appAttempt != null) { + return appAttempt.getLiveContainers().size(); + } else { + return 0; + } + } } - } ); metrics.register("variable.app." + oldAppId + ".reserved.containers", - new Gauge() { - @Override - public Integer getValue() { - SchedulerAppReport app = scheduler.getSchedulerAppInfo(appAttemptId); - return app.getReservedContainers().size(); + new Gauge() { + @Override + public Integer getValue() { + SchedulerApplicationAttempt appAttempt = + getSchedulerAppAttempt(appId); + if (appAttempt != null) { + return appAttempt.getReservedContainers().size(); + } else { + return 0; + } + } } - } ); } - - public void untrackApp(ApplicationAttemptId appAttemptId, - String oldAppId) { + + public void untrackApp(String oldAppId) { for (String m : appTrackedMetrics) { metrics.remove("variable.app." + oldAppId + "." + m); } diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerWrapper.java index 524b8bf23e1..962b137a7a5 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerWrapper.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerWrapper.java @@ -21,7 +21,6 @@ import java.util.Set; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import com.codahale.metrics.MetricRegistry; @@ -30,18 +29,16 @@ import com.codahale.metrics.MetricRegistry; @Unstable public interface SchedulerWrapper { - public MetricRegistry getMetrics(); - public SchedulerMetrics getSchedulerMetrics(); - public Set getQueueSet(); - public void setQueueSet(Set queues); - public Set getTrackedAppSet(); - public void setTrackedAppSet(Set apps); - public void addTrackedApp(ApplicationAttemptId appAttemptId, - String oldAppId); - public void removeTrackedApp(ApplicationAttemptId appAttemptId, - String oldAppId); - public void addAMRuntime(ApplicationId appId, - long traceStartTimeMS, long traceEndTimeMS, - long simulateStartTimeMS, long simulateEndTimeMS); + MetricRegistry getMetrics(); + SchedulerMetrics getSchedulerMetrics(); + Set getQueueSet(); + void setQueueSet(Set queues); + Set getTrackedAppSet(); + void setTrackedAppSet(Set apps); + void addTrackedApp(ApplicationId appId, String oldAppId); + void removeTrackedApp(String oldAppId); + void addAMRuntime(ApplicationId appId, + long traceStartTimeMS, long traceEndTimeMS, + long simulateStartTimeMS, long simulateEndTimeMS); } diff --git a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/TestAMSimulator.java b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/TestAMSimulator.java index 83482c33686..f0d8e6f507b 100644 --- a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/TestAMSimulator.java +++ b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/TestAMSimulator.java @@ -17,32 +17,43 @@ */ package org.apache.hadoop.yarn.sls.appmaster; +import com.codahale.metrics.MetricRegistry; +import org.apache.commons.io.FileUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.sls.conf.SLSConfiguration; import org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator; +import org.apache.hadoop.yarn.sls.scheduler.FairSchedulerMetrics; +import org.apache.hadoop.yarn.sls.scheduler.SchedulerWrapper; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; public class TestAMSimulator { private ResourceManager rm; private YarnConfiguration conf; + private Path metricOutputDir; @Before public void setup() { + createMetricOutputDir(); + conf = new YarnConfiguration(); + conf.set(SLSConfiguration.METRICS_OUTPUT_DIR, metricOutputDir.toString()); conf.set(YarnConfiguration.RM_SCHEDULER, "org.apache.hadoop.yarn.sls.scheduler.ResourceSchedulerWrapper"); conf.set(SLSConfiguration.RM_SCHEDULER, "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler"); - conf.setBoolean(SLSConfiguration.METRICS_SWITCH, false); + conf.setBoolean(SLSConfiguration.METRICS_SWITCH, true); rm = new ResourceManager(); rm.init(conf); rm.start(); @@ -64,14 +75,49 @@ public class TestAMSimulator { } } + private void verifySchedulerMetrics(String appId) { + SchedulerWrapper schedulerWrapper = (SchedulerWrapper) + rm.getResourceScheduler(); + MetricRegistry metricRegistry = schedulerWrapper.getMetrics(); + for (FairSchedulerMetrics.Metric metric : + FairSchedulerMetrics.Metric.values()) { + String key = "variable.app." + appId + "." + metric.getValue() + + ".memory"; + Assert.assertTrue(metricRegistry.getGauges().containsKey(key)); + Assert.assertNotNull(metricRegistry.getGauges().get(key).getValue()); + } + } + + private void createMetricOutputDir() { + Path testDir = Paths.get(System.getProperty("test.build.data")); + try { + metricOutputDir = Files.createTempDirectory(testDir, "output"); + } catch (IOException e) { + Assert.fail(e.toString()); + } + } + + private void deleteMetricOutputDir() { + try { + FileUtils.deleteDirectory(metricOutputDir.toFile()); + } catch (IOException e) { + Assert.fail(e.toString()); + } + } + @Test public void testAMSimulator() throws Exception { // Register one app MockAMSimulator app = new MockAMSimulator(); - List containers = new ArrayList(); - app.init(1, 1000, containers, rm, null, 0, 1000000l, "user1", "default", - false, "app1"); + String appId = "app1"; + String queue = "default"; + List containers = new ArrayList<>(); + app.init(1, 1000, containers, rm, null, 0, 1000000L, "user1", queue, + true, appId); app.firstStep(); + + verifySchedulerMetrics(appId); + Assert.assertEquals(1, rm.getRMContext().getRMApps().size()); Assert.assertNotNull(rm.getRMContext().getRMApps().get(app.appId)); @@ -82,5 +128,7 @@ public class TestAMSimulator { @After public void tearDown() { rm.stop(); + + deleteMetricOutputDir(); } } \ No newline at end of file From f462e1ff68d698a669406d967336bb812cf6981b Mon Sep 17 00:00:00 2001 From: Varun Saxena Date: Wed, 22 Mar 2017 04:11:53 +0530 Subject: [PATCH 109/188] YARN-5934. Fix TestTimelineWebServices.testPrimaryFilterNumericString (Akira Ajisaka via Varun Saxena) --- .../webapp/TestTimelineWebServices.java | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java index 80d801d52b4..ca78cbcbab3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java @@ -334,33 +334,6 @@ public class TestTimelineWebServices extends JerseyTestBase { verifyEntities(response.getEntity(TimelineEntities.class)); } - @Test - public void testPrimaryFilterNumericString() { - // without quotes, 123abc is interpreted as the number 123, - // which finds no entities - WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("timeline") - .path("type_1").queryParam("primaryFilter", "other:123abc") - .accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); - assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, - response.getType().toString()); - assertEquals(0, response.getEntity(TimelineEntities.class).getEntities() - .size()); - } - - @Test - public void testPrimaryFilterNumericStringWithQuotes() { - WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("timeline") - .path("type_1").queryParam("primaryFilter", "other:\"123abc\"") - .accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); - assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, - response.getType().toString()); - verifyEntities(response.getEntity(TimelineEntities.class)); - } - @Test public void testSecondaryFilters() { WebResource r = resource(); From ed2d03f2a5318a88d0eced8476127f303eb1d1d9 Mon Sep 17 00:00:00 2001 From: John Zhuge Date: Wed, 22 Mar 2017 22:15:24 -0700 Subject: [PATCH 110/188] HDFS-11561. HttpFS doc errors. Contributed by Yuanbo Liu. --- .../src/site/markdown/index.md | 8 ++++---- .../fs/http/server/TestHttpFSServer.java | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/index.md b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/index.md index 2e54431dedf..6eef9e7d30e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/index.md +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/site/markdown/index.md @@ -36,13 +36,13 @@ HttpFS itself is Java Jetty web-application. HttpFS HTTP web-service API calls are HTTP REST calls that map to a HDFS file system operation. For example, using the `curl` Unix command: -* `$ curl http://httpfs-host:14000/webhdfs/v1/user/foo/README.txt` returns the contents of the HDFS `/user/foo/README.txt` file. +* `$ curl 'http://httpfs-host:14000/webhdfs/v1/user/foo/README.txt?op=OPEN&user.name=foo'` returns the contents of the HDFS `/user/foo/README.txt` file. -* `$ curl http://httpfs-host:14000/webhdfs/v1/user/foo?op=list` returns the contents of the HDFS `/user/foo` directory in JSON format. +* `$ curl 'http://httpfs-host:14000/webhdfs/v1/user/foo?op=LISTSTATUS&user.name=foo'` returns the contents of the HDFS `/user/foo` directory in JSON format. -* `$ curl http://httpfs-host:14000/webhdfs/v1/user/foo?op=GETTRASHROOT` returns the path `/user/foo/.Trash`, if `/` is an encrypted zone, returns the path `/.Trash/foo`. See [more details](../hadoop-project-dist/hadoop-hdfs/TransparentEncryption.html#Rename_and_Trash_considerations) about trash path in an encrypted zone. +* `$ curl 'http://httpfs-host:14000/webhdfs/v1/user/foo?op=GETTRASHROOT&user.name=foo'` returns the path `/user/foo/.Trash`, if `/` is an encrypted zone, returns the path `/.Trash/foo`. See [more details](../hadoop-project-dist/hadoop-hdfs/TransparentEncryption.html#Rename_and_Trash_considerations) about trash path in an encrypted zone. -* `$ curl -X POST http://httpfs-host:14000/webhdfs/v1/user/foo/bar?op=mkdirs` creates the HDFS `/user/foo.bar` directory. +* `$ curl -X POST 'http://httpfs-host:14000/webhdfs/v1/user/foo/bar?op=MKDIRS&user.name=foo'` creates the HDFS `/user/foo/bar` directory. User and Developer Documentation -------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java index fcbddc48198..91d22c832ca 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java @@ -223,6 +223,24 @@ public class TestHttpFSServer extends HFSTestCase { reader.close(); } + @Test + @TestDir + @TestJetty + @TestHdfs + public void testMkdirs() throws Exception { + createHttpFSServer(false); + + String user = HadoopUsersConfTestHelper.getHadoopUsers()[0]; + URL url = new URL(TestJettyHelper.getJettyURL(), MessageFormat.format( + "/webhdfs/v1/tmp/sub-tmp?user.name={0}&op=MKDIRS", user)); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("PUT"); + conn.connect(); + Assert.assertEquals(conn.getResponseCode(), HttpURLConnection.HTTP_OK); + + getStatus("/tmp/sub-tmp", "LISTSTATUS"); + } + @Test @TestDir @TestJetty From 59d69257a888347f0fb9c51bb000afc986b64f98 Mon Sep 17 00:00:00 2001 From: Zhe Zhang Date: Wed, 22 Mar 2017 22:21:30 -0700 Subject: [PATCH 111/188] HADOOP-9631. ViewFs should use underlying FileSystem's server side defaults. Contributed by Lohit Vijayarenu and Erik Krogen. --- .../apache/hadoop/fs/AbstractFileSystem.java | 16 +++++- .../java/org/apache/hadoop/fs/ChecksumFs.java | 13 ++--- .../hadoop/fs/DelegateToFileSystem.java | 6 +++ .../java/org/apache/hadoop/fs/FilterFs.java | 5 ++ .../java/org/apache/hadoop/fs/ftp/FtpFs.java | 7 +++ .../apache/hadoop/fs/local/RawLocalFs.java | 8 +++ .../apache/hadoop/fs/viewfs/ChRootedFs.java | 6 +++ .../org/apache/hadoop/fs/viewfs/ViewFs.java | 20 +++++++- .../apache/hadoop/fs/TestAfsCheckPath.java | 1 + .../hadoop/fs/viewfs/ViewFsBaseTest.java | 49 +++++++++++++++++++ .../main/java/org/apache/hadoop/fs/Hdfs.java | 6 +++ 11 files changed, 128 insertions(+), 9 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java index d1e7a1b7383..ef684372744 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java @@ -450,9 +450,21 @@ public abstract class AbstractFileSystem { * @return server default configuration values * * @throws IOException an I/O error occurred + * @deprecated use {@link #getServerDefaults(Path)} instead */ + @Deprecated public abstract FsServerDefaults getServerDefaults() throws IOException; + /** + * Return a set of server default configuration values based on path. + * @param f path to fetch server defaults + * @return server default configuration values for path + * @throws IOException an I/O error occurred + */ + public FsServerDefaults getServerDefaults(final Path f) throws IOException { + return getServerDefaults(); + } + /** * Return the fully-qualified path of path f resolving the path * through any internal symlinks or mount point @@ -548,7 +560,7 @@ public abstract class AbstractFileSystem { } - FsServerDefaults ssDef = getServerDefaults(); + FsServerDefaults ssDef = getServerDefaults(f); if (ssDef.getBlockSize() % ssDef.getBytesPerChecksum() != 0) { throw new IOException("Internal error: default blockSize is" + " not a multiple of default bytesPerChecksum "); @@ -626,7 +638,7 @@ public abstract class AbstractFileSystem { */ public FSDataInputStream open(final Path f) throws AccessControlException, FileNotFoundException, UnresolvedLinkException, IOException { - return open(f, getServerDefaults().getFileBufferSize()); + return open(f, getServerDefaults(f).getFileBufferSize()); } /** diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java index 397203368e7..0a8cc737130 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java @@ -57,7 +57,7 @@ public abstract class ChecksumFs extends FilterFs { throws IOException, URISyntaxException { super(theFs); defaultBytesPerChecksum = - getMyFs().getServerDefaults().getBytesPerChecksum(); + getMyFs().getServerDefaults(new Path("/")).getBytesPerChecksum(); } /** @@ -96,9 +96,10 @@ public abstract class ChecksumFs extends FilterFs { return defaultBytesPerChecksum; } - private int getSumBufferSize(int bytesPerSum, int bufferSize) + private int getSumBufferSize(int bytesPerSum, int bufferSize, Path file) throws IOException { - int defaultBufferSize = getMyFs().getServerDefaults().getFileBufferSize(); + int defaultBufferSize = getMyFs().getServerDefaults(file) + .getFileBufferSize(); int proportionalBufferSize = bufferSize / bytesPerSum; return Math.max(bytesPerSum, Math.max(proportionalBufferSize, defaultBufferSize)); @@ -121,7 +122,7 @@ public abstract class ChecksumFs extends FilterFs { public ChecksumFSInputChecker(ChecksumFs fs, Path file) throws IOException, UnresolvedLinkException { - this(fs, file, fs.getServerDefaults().getFileBufferSize()); + this(fs, file, fs.getServerDefaults(file).getFileBufferSize()); } public ChecksumFSInputChecker(ChecksumFs fs, Path file, int bufferSize) @@ -132,7 +133,7 @@ public abstract class ChecksumFs extends FilterFs { Path sumFile = fs.getChecksumFile(file); try { int sumBufferSize = fs.getSumBufferSize(fs.getBytesPerSum(), - bufferSize); + bufferSize, file); sums = fs.getRawFs().open(sumFile, sumBufferSize); byte[] version = new byte[CHECKSUM_VERSION.length]; @@ -353,7 +354,7 @@ public abstract class ChecksumFs extends FilterFs { // Now create the chekcsumfile; adjust the buffsize int bytesPerSum = fs.getBytesPerSum(); - int sumBufferSize = fs.getSumBufferSize(bytesPerSum, bufferSize); + int sumBufferSize = fs.getSumBufferSize(bytesPerSum, bufferSize, file); this.sums = fs.getRawFs().createInternal(fs.getChecksumFile(file), EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE), absolutePermission, sumBufferSize, replication, blockSize, progress, diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java index d2550dc2f5d..a5ab75e4744 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java @@ -149,10 +149,16 @@ public abstract class DelegateToFileSystem extends AbstractFileSystem { } @Override + @Deprecated public FsServerDefaults getServerDefaults() throws IOException { return fsImpl.getServerDefaults(); } + @Override + public FsServerDefaults getServerDefaults(final Path f) throws IOException { + return fsImpl.getServerDefaults(f); + } + @Override public Path getHomeDirectory() { return fsImpl.getHomeDirectory(); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java index d7e313ffe8c..6b1093e7969 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java @@ -147,10 +147,15 @@ public abstract class FilterFs extends AbstractFileSystem { } @Override + @Deprecated public FsServerDefaults getServerDefaults() throws IOException { return myFs.getServerDefaults(); } + @Override + public FsServerDefaults getServerDefaults(final Path f) throws IOException { + return myFs.getServerDefaults(f); + } @Override public Path resolvePath(final Path p) throws FileNotFoundException, diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpFs.java index 0a97375d9cd..6d54e36f674 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpFs.java @@ -29,6 +29,7 @@ import org.apache.hadoop.fs.AbstractFileSystem; import org.apache.hadoop.fs.DelegateToFileSystem; import org.apache.hadoop.fs.FsConstants; import org.apache.hadoop.fs.FsServerDefaults; +import org.apache.hadoop.fs.Path; /** * The FtpFs implementation of AbstractFileSystem. @@ -57,7 +58,13 @@ public class FtpFs extends DelegateToFileSystem { } @Override + @Deprecated public FsServerDefaults getServerDefaults() throws IOException { return FtpConfigKeys.getServerDefaults(); } + + @Override + public FsServerDefaults getServerDefaults(final Path f) throws IOException { + return FtpConfigKeys.getServerDefaults(); + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java index 6cb2792eebc..8aebe027b18 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.AbstractFileSystem; import org.apache.hadoop.fs.DelegateToFileSystem; import org.apache.hadoop.fs.FsConstants; import org.apache.hadoop.fs.FsServerDefaults; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RawLocalFileSystem; /** @@ -63,6 +64,13 @@ public class RawLocalFs extends DelegateToFileSystem { } @Override + public FsServerDefaults getServerDefaults(final Path f) + throws IOException { + return LocalConfigKeys.getServerDefaults(); + } + + @Override + @Deprecated public FsServerDefaults getServerDefaults() throws IOException { return LocalConfigKeys.getServerDefaults(); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java index b7e47c21f13..d77ad8b96e3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java @@ -223,10 +223,16 @@ class ChRootedFs extends AbstractFileSystem { } @Override + @Deprecated public FsServerDefaults getServerDefaults() throws IOException { return myFs.getServerDefaults(); } + @Override + public FsServerDefaults getServerDefaults(final Path f) throws IOException { + return myFs.getServerDefaults(fullPath(f)); + } + @Override public int getUriDefaultPort() { return myFs.getUriDefaultPort(); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java index 1a54a81a0f1..3a34a9102ec 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java @@ -240,10 +240,22 @@ public class ViewFs extends AbstractFileSystem { } @Override + @Deprecated public FsServerDefaults getServerDefaults() throws IOException { return LocalConfigKeys.getServerDefaults(); } + @Override + public FsServerDefaults getServerDefaults(final Path f) throws IOException { + InodeTree.ResolveResult res; + try { + res = fsState.resolve(getUriPath(f), true); + } catch (FileNotFoundException fnfe) { + return LocalConfigKeys.getServerDefaults(); + } + return res.targetFileSystem.getServerDefaults(res.remainingPath); + } + @Override public int getUriDefaultPort() { return -1; @@ -927,8 +939,14 @@ public class ViewFs extends AbstractFileSystem { } @Override + @Deprecated public FsServerDefaults getServerDefaults() throws IOException { - throw new IOException("FsServerDefaults not implemented yet"); + return LocalConfigKeys.getServerDefaults(); + } + + @Override + public FsServerDefaults getServerDefaults(final Path f) throws IOException { + return LocalConfigKeys.getServerDefaults(); } @Override diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java index 3bd14f1495a..da429ffe960 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java @@ -117,6 +117,7 @@ public class TestAfsCheckPath { } @Override + @Deprecated public FsServerDefaults getServerDefaults() throws IOException { // deliberately empty return null; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java index 8f6df32e777..50237d1ed01 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java @@ -39,6 +39,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; +import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -46,8 +47,10 @@ import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.AbstractFileSystem; import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileContextTestHelper; +import org.apache.hadoop.fs.FsServerDefaults; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.FileContextTestHelper.fileType; @@ -55,6 +58,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FsConstants; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnresolvedLinkException; +import org.apache.hadoop.fs.local.LocalConfigKeys; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.AclUtil; @@ -839,6 +843,51 @@ abstract public class ViewFsBaseTest { }); } + @Test + public void testRespectsServerDefaults() throws Exception { + FsServerDefaults targetDefs = + fcTarget.getDefaultFileSystem().getServerDefaults(new Path("/")); + FsServerDefaults viewDefs = + fcView.getDefaultFileSystem().getServerDefaults(new Path("/data")); + assertEquals(targetDefs.getReplication(), viewDefs.getReplication()); + assertEquals(targetDefs.getBlockSize(), viewDefs.getBlockSize()); + assertEquals(targetDefs.getBytesPerChecksum(), + viewDefs.getBytesPerChecksum()); + assertEquals(targetDefs.getFileBufferSize(), + viewDefs.getFileBufferSize()); + assertEquals(targetDefs.getWritePacketSize(), + viewDefs.getWritePacketSize()); + assertEquals(targetDefs.getEncryptDataTransfer(), + viewDefs.getEncryptDataTransfer()); + assertEquals(targetDefs.getTrashInterval(), viewDefs.getTrashInterval()); + assertEquals(targetDefs.getChecksumType(), viewDefs.getChecksumType()); + + fcView.create(new Path("/data/file"), EnumSet.of(CreateFlag.CREATE)) + .close(); + FileStatus stat = + fcTarget.getFileStatus(new Path(targetTestRoot, "data/file")); + assertEquals(targetDefs.getReplication(), stat.getReplication()); + } + + @Test + public void testServerDefaultsInternalDir() throws Exception { + FsServerDefaults localDefs = LocalConfigKeys.getServerDefaults(); + FsServerDefaults viewDefs = fcView + .getDefaultFileSystem().getServerDefaults(new Path("/internalDir")); + assertEquals(localDefs.getReplication(), viewDefs.getReplication()); + assertEquals(localDefs.getBlockSize(), viewDefs.getBlockSize()); + assertEquals(localDefs.getBytesPerChecksum(), + viewDefs.getBytesPerChecksum()); + assertEquals(localDefs.getFileBufferSize(), + viewDefs.getFileBufferSize()); + assertEquals(localDefs.getWritePacketSize(), + viewDefs.getWritePacketSize()); + assertEquals(localDefs.getEncryptDataTransfer(), + viewDefs.getEncryptDataTransfer()); + assertEquals(localDefs.getTrashInterval(), viewDefs.getTrashInterval()); + assertEquals(localDefs.getChecksumType(), viewDefs.getChecksumType()); + } + // Confirm that listLocatedStatus is delegated properly to the underlying // AbstractFileSystem to allow for optimizations @Test diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java index 82ee41a1221..645f1ad833a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java @@ -155,10 +155,16 @@ public class Hdfs extends AbstractFileSystem { } @Override + @Deprecated public FsServerDefaults getServerDefaults() throws IOException { return dfs.getServerDefaults(); } + @Override + public FsServerDefaults getServerDefaults(final Path f) throws IOException { + return dfs.getServerDefaults(); + } + @Override public RemoteIterator listLocatedStatus( final Path p) From a5a4867f3b193a137a6260d539da7e21f02ffab3 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Thu, 23 Mar 2017 12:54:54 +0000 Subject: [PATCH 112/188] HADOOP-14120 needless S3AFileSystem.setOptionalPutRequestParameters in S3ABlockOutputStream putObject(). Contributed by Yuanbo Liu (cherry picked from commit 20878d052cebc715c2494a97362fdff08885a77f) --- .../java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java | 5 ++--- .../main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java index 1b0929b5c80..3fbdcb06ba8 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java @@ -378,9 +378,8 @@ class S3ABlockOutputStream extends OutputStream { int size = block.dataSize(); final S3ADataBlocks.BlockUploadData uploadData = block.startUpload(); final PutObjectRequest putObjectRequest = uploadData.hasFile() ? - writeOperationHelper.newPutRequest(uploadData.getFile()) - : writeOperationHelper.newPutRequest(uploadData.getUploadStream(), size); - fs.setOptionalPutRequestParameters(putObjectRequest); + writeOperationHelper.newPutRequest(uploadData.getFile()) : + writeOperationHelper.newPutRequest(uploadData.getUploadStream(), size); long transferQueueTime = now(); BlockUploadProgress callback = new BlockUploadProgress( diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 6c8941253e4..57fbb85dcff 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -1870,7 +1870,7 @@ public class S3AFileSystem extends FileSystem { } } - protected void setOptionalPutRequestParameters(PutObjectRequest request) { + private void setOptionalPutRequestParameters(PutObjectRequest request) { switch (serverSideEncryptionAlgorithm) { case SSE_KMS: request.setSSEAwsKeyManagementParams(generateSSEAwsKeyParams()); From 595f62e362c08704d6fb692e21c97b512bc7ec49 Mon Sep 17 00:00:00 2001 From: Ravi Prakash Date: Thu, 23 Mar 2017 09:28:10 -0700 Subject: [PATCH 113/188] HADOOP-14213. Move Configuration runtime check for hadoop-site.xml to initialization. Contributed by Jonathan Eagles --- .../org/apache/hadoop/conf/Configuration.java | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index bade06ea307..a9c8d9cc934 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -668,21 +668,24 @@ public class Configuration implements Iterable>, } } - static{ - //print deprecation warning if hadoop-site.xml is found in classpath + static { + // Add default resources + addDefaultResource("core-default.xml"); + addDefaultResource("core-site.xml"); + + // print deprecation warning if hadoop-site.xml is found in classpath ClassLoader cL = Thread.currentThread().getContextClassLoader(); if (cL == null) { cL = Configuration.class.getClassLoader(); } - if(cL.getResource("hadoop-site.xml")!=null) { + if (cL.getResource("hadoop-site.xml") != null) { LOG.warn("DEPRECATED: hadoop-site.xml found in the classpath. " + "Usage of hadoop-site.xml is deprecated. Instead use core-site.xml, " + "mapred-site.xml and hdfs-site.xml to override properties of " + "core-default.xml, mapred-default.xml and hdfs-default.xml " + "respectively"); + addDefaultResource("hadoop-site.xml"); } - addDefaultResource("core-default.xml"); - addDefaultResource("core-site.xml"); } private Properties properties; @@ -2638,11 +2641,6 @@ public class Configuration implements Iterable>, for (String resource : defaultResources) { loadResource(properties, new Resource(resource), quiet); } - - //support the hadoop-site.xml as a deprecated case - if(getResource("hadoop-site.xml")!=null) { - loadResource(properties, new Resource("hadoop-site.xml"), quiet); - } } for (int i = 0; i < resources.size(); i++) { From 2e30aa72e01de7b5774fcb312406a393221e0908 Mon Sep 17 00:00:00 2001 From: Mingliang Liu Date: Tue, 28 Feb 2017 14:51:32 -0800 Subject: [PATCH 114/188] HADOOP-14135. Remove URI parameter in AWSCredentialProvider constructors. Contributed by Mingliang Liu --- .../apache/hadoop/fs/s3a/S3AFileSystem.java | 2 +- .../org/apache/hadoop/fs/s3a/S3AUtils.java | 34 +++++--------- .../apache/hadoop/fs/s3a/S3ClientFactory.java | 7 ++- .../fs/s3a/SimpleAWSCredentialsProvider.java | 3 +- .../s3a/TemporaryAWSCredentialsProvider.java | 3 +- .../s3a/ITestS3AAWSCredentialsProvider.java | 44 +++++++++++++++++-- .../fs/s3a/ITestS3ATemporaryCredentials.java | 2 +- .../hadoop/fs/s3a/MockS3ClientFactory.java | 2 +- .../fs/s3a/TestS3AAWSCredentialsProvider.java | 16 +++---- 9 files changed, 69 insertions(+), 44 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 57fbb85dcff..b17281be8f8 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -186,7 +186,7 @@ public class S3AFileSystem extends FileSystem { S3_CLIENT_FACTORY_IMPL, DEFAULT_S3_CLIENT_FACTORY_IMPL, S3ClientFactory.class); s3 = ReflectionUtils.newInstance(s3ClientFactoryClass, conf) - .createS3Client(name, uri); + .createS3Client(name); maxKeys = intOption(conf, MAX_PAGING_KEYS, DEFAULT_MAX_PAGING_KEYS, 1); listing = new Listing(this); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java index 84f3c9964e2..6a11699ba5f 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java @@ -318,15 +318,12 @@ public final class S3AUtils { * Create the AWS credentials from the providers and the URI. * @param binding Binding URI, may contain user:pass login details * @param conf filesystem configuration - * @param fsURI fS URI —after any login details have been stripped. * @return a credentials provider list * @throws IOException Problems loading the providers (including reading * secrets from credential files). */ public static AWSCredentialProviderList createAWSCredentialProviderSet( - URI binding, - Configuration conf, - URI fsURI) throws IOException { + URI binding, Configuration conf) throws IOException { AWSCredentialProviderList credentials = new AWSCredentialProviderList(); Class[] awsClasses; @@ -351,11 +348,12 @@ public final class S3AUtils { SharedInstanceProfileCredentialsProvider.class.getName()); aClass = SharedInstanceProfileCredentialsProvider.class; } - credentials.add(createAWSCredentialProvider(conf, - aClass, - fsURI)); + credentials.add(createAWSCredentialProvider(conf, aClass)); } } + // make sure the logging message strips out any auth details + LOG.debug("For URI {}, using credentials {}", + S3xLoginHelper.toString(binding), credentials); return credentials; } @@ -365,8 +363,8 @@ public final class S3AUtils { * attempted in order: * *

      - *
    1. a public constructor accepting java.net.URI and - * org.apache.hadoop.conf.Configuration
      2. + *
    2. a public constructor accepting + * org.apache.hadoop.conf.Configuration
      3. *
    3. a public static method named getInstance that accepts no * arguments and returns an instance of * com.amazonaws.auth.AWSCredentialsProvider, or
      4. @@ -375,14 +373,11 @@ public final class S3AUtils { * * @param conf configuration * @param credClass credential class - * @param uri URI of the FS * @return the instantiated class * @throws IOException on any instantiation failure. */ static AWSCredentialsProvider createAWSCredentialProvider( - Configuration conf, - Class credClass, - URI uri) throws IOException { + Configuration conf, Class credClass) throws IOException { AWSCredentialsProvider credentials = null; String className = credClass.getName(); if (!AWSCredentialsProvider.class.isAssignableFrom(credClass)) { @@ -394,11 +389,10 @@ public final class S3AUtils { LOG.debug("Credential provider class is {}", className); try { - // new X(uri, conf) - Constructor cons = getConstructor(credClass, URI.class, - Configuration.class); + // new X(conf) + Constructor cons = getConstructor(credClass, Configuration.class); if (cons != null) { - credentials = (AWSCredentialsProvider)cons.newInstance(uri, conf); + credentials = (AWSCredentialsProvider)cons.newInstance(conf); return credentials; } @@ -420,16 +414,12 @@ public final class S3AUtils { // no supported constructor or factory method found throw new IOException(String.format("%s " + CONSTRUCTOR_EXCEPTION + ". A class specified in %s must provide a public constructor " - + "accepting URI and Configuration, or a public factory method named " + + "accepting Configuration, or a public factory method named " + "getInstance that accepts no arguments, or a public default " + "constructor.", className, AWS_CREDENTIALS_PROVIDER)); } catch (ReflectiveOperationException | IllegalArgumentException e) { // supported constructor or factory method found, but the call failed throw new IOException(className + " " + INSTANTIATION_EXCEPTION +".", e); - } finally { - if (credentials != null) { - LOG.debug("Using {} for {}.", credentials, uri); - } } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java index 871322db455..d4e09e300d7 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java @@ -52,11 +52,10 @@ interface S3ClientFactory { * because both values may be useful in logging. * * @param name raw input S3A file system URI - * @param uri validated form of S3A file system URI * @return S3 client * @throws IOException IO problem */ - AmazonS3 createS3Client(URI name, URI uri) throws IOException; + AmazonS3 createS3Client(URI name) throws IOException; /** * The default factory implementation, which calls the AWS SDK to configure @@ -68,10 +67,10 @@ interface S3ClientFactory { private static final Logger LOG = S3AFileSystem.LOG; @Override - public AmazonS3 createS3Client(URI name, URI uri) throws IOException { + public AmazonS3 createS3Client(URI name) throws IOException { Configuration conf = getConf(); AWSCredentialsProvider credentials = - createAWSCredentialProviderSet(name, conf, uri); + createAWSCredentialProviderSet(name, conf); ClientConfiguration awsConf = new ClientConfiguration(); initConnectionSettings(conf, awsConf); initProxySupport(conf, awsConf); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java index 13c139d5e0a..ec372bd6136 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java @@ -28,7 +28,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.ProviderUtils; import java.io.IOException; -import java.net.URI; import static org.apache.hadoop.fs.s3a.Constants.ACCESS_KEY; import static org.apache.hadoop.fs.s3a.Constants.SECRET_KEY; @@ -51,7 +50,7 @@ public class SimpleAWSCredentialsProvider implements AWSCredentialsProvider { private String secretKey; private IOException lookupIOE; - public SimpleAWSCredentialsProvider(URI uri, Configuration conf) { + public SimpleAWSCredentialsProvider(Configuration conf) { try { Configuration c = ProviderUtils.excludeIncompatibleCredentialProviders( conf, S3AFileSystem.class); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java index 883ae86394f..22b23a4ad7b 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java @@ -24,7 +24,6 @@ import com.amazonaws.auth.AWSCredentials; import org.apache.commons.lang.StringUtils; import java.io.IOException; -import java.net.URI; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -51,7 +50,7 @@ public class TemporaryAWSCredentialsProvider implements AWSCredentialsProvider { private String sessionToken; private IOException lookupIOE; - public TemporaryAWSCredentialsProvider(URI uri, Configuration conf) { + public TemporaryAWSCredentialsProvider(Configuration conf) { try { Configuration c = ProviderUtils.excludeIncompatibleCredentialProviders( conf, S3AFileSystem.class); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java index 99454f4d38e..22c4f7ee41f 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java @@ -19,13 +19,14 @@ package org.apache.hadoop.fs.s3a; import java.io.IOException; -import java.net.URI; import java.nio.file.AccessDeniedException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.test.GenericTestUtils; + import org.junit.Rule; import org.junit.Test; import org.junit.rules.Timeout; @@ -40,6 +41,7 @@ import org.slf4j.LoggerFactory; import static org.apache.hadoop.fs.s3a.Constants.*; import static org.apache.hadoop.fs.s3a.S3ATestConstants.*; +import static org.apache.hadoop.fs.s3a.S3AUtils.*; import static org.junit.Assert.*; /** @@ -66,6 +68,42 @@ public class ITestS3AAWSCredentialsProvider { } } + /** + * A bad CredentialsProvider which has no suitable constructor. + * + * This class does not provide a public constructor accepting Configuration, + * or a public factory method named getInstance that accepts no arguments, + * or a public default constructor. + */ + static class BadCredentialsProviderConstructor + implements AWSCredentialsProvider { + + @SuppressWarnings("unused") + public BadCredentialsProviderConstructor(String fsUri, Configuration conf) { + } + + @Override + public AWSCredentials getCredentials() { + return new BasicAWSCredentials("dummy_key", "dummy_secret"); + } + + @Override + public void refresh() { + } + } + + @Test + public void testBadCredentialsConstructor() throws Exception { + Configuration conf = new Configuration(); + conf.set(AWS_CREDENTIALS_PROVIDER, + BadCredentialsProviderConstructor.class.getName()); + try { + createFailingFS(conf); + } catch (IOException e) { + GenericTestUtils.assertExceptionContains(CONSTRUCTOR_EXCEPTION, e); + } + } + /** * Create a filesystem, expect it to fail by raising an IOException. * Raises an assertion exception if in fact the FS does get instantiated. @@ -81,7 +119,7 @@ public class ITestS3AAWSCredentialsProvider { static class BadCredentialsProvider implements AWSCredentialsProvider { @SuppressWarnings("unused") - public BadCredentialsProvider(URI name, Configuration conf) { + public BadCredentialsProvider(Configuration conf) { } @Override @@ -108,7 +146,7 @@ public class ITestS3AAWSCredentialsProvider { static class GoodCredentialsProvider extends AWSCredentialsProviderChain { @SuppressWarnings("unused") - public GoodCredentialsProvider(URI name, Configuration conf) { + public GoodCredentialsProvider(Configuration conf) { super(new BasicAWSCredentialsProvider(conf.get(ACCESS_KEY), conf.get(SECRET_KEY)), InstanceProfileCredentialsProvider.getInstance()); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java index 84aad3c9b7f..4abe2b79e9e 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java @@ -126,7 +126,7 @@ public class ITestS3ATemporaryCredentials extends AbstractS3ATestBase { conf.set(SECRET_KEY, "secretkey"); conf.set(SESSION_TOKEN, ""); TemporaryAWSCredentialsProvider provider - = new TemporaryAWSCredentialsProvider(getFileSystem().getUri(), conf); + = new TemporaryAWSCredentialsProvider(conf); try { AWSCredentials credentials = provider.getCredentials(); fail("Expected a CredentialInitializationException," diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java index 41f04ee728b..9e0a5e42b62 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java @@ -31,7 +31,7 @@ import com.amazonaws.services.s3.AmazonS3; public class MockS3ClientFactory implements S3ClientFactory { @Override - public AmazonS3 createS3Client(URI name, URI uri) { + public AmazonS3 createS3Client(URI name) { String bucket = name.getHost(); AmazonS3 s3 = mock(AmazonS3.class); when(s3.doesBucketExist(bucket)).thenReturn(true); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java index c29d7254b84..33740c8ab93 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java @@ -93,7 +93,7 @@ public class TestS3AAWSCredentialsProvider { URI uri = testFile.toUri(); AWSCredentialProviderList list = S3AUtils.createAWSCredentialProviderSet( - uri, conf, uri); + uri, conf); List> expectedClasses = Arrays.asList( TemporaryAWSCredentialsProvider.class, @@ -107,9 +107,9 @@ public class TestS3AAWSCredentialsProvider { URI uri1 = new URI("s3a://bucket1"), uri2 = new URI("s3a://bucket2"); Configuration conf = new Configuration(); AWSCredentialProviderList list1 = S3AUtils.createAWSCredentialProviderSet( - uri1, conf, uri1); + uri1, conf); AWSCredentialProviderList list2 = S3AUtils.createAWSCredentialProviderSet( - uri2, conf, uri2); + uri2, conf); List> expectedClasses = Arrays.asList( BasicAWSCredentialsProvider.class, @@ -132,9 +132,9 @@ public class TestS3AAWSCredentialsProvider { AnonymousAWSCredentialsProvider.class); conf.set(AWS_CREDENTIALS_PROVIDER, buildClassListString(expectedClasses)); AWSCredentialProviderList list1 = S3AUtils.createAWSCredentialProviderSet( - uri1, conf, uri1); + uri1, conf); AWSCredentialProviderList list2 = S3AUtils.createAWSCredentialProviderSet( - uri2, conf, uri2); + uri2, conf); assertCredentialProviders(expectedClasses, list1); assertCredentialProviders(expectedClasses, list2); assertSameInstanceProfileCredentialsProvider(list1.getProviders().get(1), @@ -150,9 +150,9 @@ public class TestS3AAWSCredentialsProvider { InstanceProfileCredentialsProvider.class); conf.set(AWS_CREDENTIALS_PROVIDER, buildClassListString(expectedClasses)); AWSCredentialProviderList list1 = S3AUtils.createAWSCredentialProviderSet( - uri1, conf, uri1); + uri1, conf); AWSCredentialProviderList list2 = S3AUtils.createAWSCredentialProviderSet( - uri2, conf, uri2); + uri2, conf); assertCredentialProviders(expectedClasses, list1); assertCredentialProviders(expectedClasses, list2); assertSameInstanceProfileCredentialsProvider(list1.getProviders().get(0), @@ -226,7 +226,7 @@ public class TestS3AAWSCredentialsProvider { conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE)); expectException(IOException.class, expectedErrorText); URI uri = testFile.toUri(); - S3AUtils.createAWSCredentialProviderSet(uri, conf, uri); + S3AUtils.createAWSCredentialProviderSet(uri, conf); } /** From 03955612b77f250d53b8f7e4268338e5b78a3e66 Mon Sep 17 00:00:00 2001 From: Chris Douglas Date: Thu, 23 Mar 2017 12:00:52 -0700 Subject: [PATCH 115/188] HADOOP-14196. Azure Data Lake doc is missing required config entry. Contributed by Atul Sikaria --- .../hadoop-azure-datalake/src/site/markdown/index.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md index 3a1625375be..3c4c70e9803 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md +++ b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md @@ -132,6 +132,11 @@ service associated with the client id. See [*Active Directory Library For Java*] Add the following properties to your `core-site.xml` ```xml + + dfs.adls.oauth2.access.token.provider.type + ClientCredential + + dfs.adls.oauth2.refresh.url TOKEN ENDPOINT FROM STEP 7 ABOVE From c55195588f5853b16d34cb6389cc6f84acc3edea Mon Sep 17 00:00:00 2001 From: Chris Douglas Date: Thu, 23 Mar 2017 12:17:27 -0700 Subject: [PATCH 116/188] HADOOP-14197. Fix ADLS doc for credential provider. Contributed by John Zhuge --- .../src/site/markdown/index.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md index 3c4c70e9803..de8fda25d17 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md +++ b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md @@ -166,15 +166,19 @@ For additional reading on the credential provider API, see ##### Provisioning ```bash -hadoop credential create dfs.adls.oauth2.refresh.token -value 123 +hadoop credential create dfs.adls.oauth2.client.id -value 123 -provider localjceks://file/home/foo/adls.jceks -hadoop credential create dfs.adls.oauth2.credential -value 123 +hadoop credential create dfs.adls.oauth2.refresh.token -value 123 -provider localjceks://file/home/foo/adls.jceks ``` ##### Configuring core-site.xml or command line property ```xml + + dfs.adls.oauth2.access.token.provider.type + RefreshToken + hadoop.security.credential.provider.path localjceks://file/home/foo/adls.jceks @@ -186,9 +190,10 @@ hadoop credential create dfs.adls.oauth2.credential -value 123 ```bash hadoop distcp - [-D hadoop.security.credential.provider.path=localjceks://file/home/user/adls.jceks] - hdfs://:9001/user/foo/007020615 - adl://.azuredatalakestore.net/testDir/ + [-D dfs.adls.oauth2.access.token.provider.type=RefreshToken + -D hadoop.security.credential.provider.path=localjceks://file/home/user/adls.jceks] + hdfs://:9001/user/foo/srcDir + adl://.azuredatalakestore.net/tgtDir/ ``` NOTE: You may optionally add the provider path property to the `distcp` command From d35e79abc2fee7153a6168e6088f100de59d8c81 Mon Sep 17 00:00:00 2001 From: Mingliang Liu Date: Thu, 23 Mar 2017 11:33:06 -0700 Subject: [PATCH 117/188] HADOOP-14214. DomainSocketWatcher::add()/delete() should not self interrupt while looping await(). Contributed by Mingliang Liu --- .../apache/hadoop/net/unix/DomainSocketWatcher.java | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java index ad2fbfb2882..e1bcf7e20c3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/unix/DomainSocketWatcher.java @@ -321,11 +321,7 @@ public final class DomainSocketWatcher implements Closeable { toAdd.add(entry); kick(); while (true) { - try { - processedCond.await(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } + processedCond.awaitUninterruptibly(); if (!toAdd.contains(entry)) { break; } @@ -347,11 +343,7 @@ public final class DomainSocketWatcher implements Closeable { toRemove.put(sock.fd, sock); kick(); while (true) { - try { - processedCond.await(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } + processedCond.awaitUninterruptibly(); if (!toRemove.containsKey(sock.fd)) { break; } From 128015584d69492806fd1700c8f840d78aa9c729 Mon Sep 17 00:00:00 2001 From: John Zhuge Date: Thu, 23 Mar 2017 13:56:19 -0700 Subject: [PATCH 118/188] HADOOP-14195. CredentialProviderFactory$getProviders is not thread-safe. Contributed by Vihang Karajgaonkar. --- .../alias/CredentialProviderFactory.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialProviderFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialProviderFactory.java index 3c3f79f4c3d..d1e3eb5437b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialProviderFactory.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialProviderFactory.java @@ -56,12 +56,16 @@ public abstract class CredentialProviderFactory { try { URI uri = new URI(path); boolean found = false; - for(CredentialProviderFactory factory: serviceLoader) { - CredentialProvider kp = factory.createProvider(uri, conf); - if (kp != null) { - result.add(kp); - found = true; - break; + // Iterate serviceLoader in a synchronized block since + // serviceLoader iterator is not thread-safe. + synchronized (serviceLoader) { + for (CredentialProviderFactory factory : serviceLoader) { + CredentialProvider kp = factory.createProvider(uri, conf); + if (kp != null) { + result.add(kp); + found = true; + break; + } } } if (!found) { From ab759e91b746fbd1d8d70f45e896a9f4bd7abf7e Mon Sep 17 00:00:00 2001 From: John Zhuge Date: Thu, 23 Mar 2017 23:31:27 -0700 Subject: [PATCH 119/188] HDFS-10649. Remove unused PermissionStatus#applyUMask. Contributed by Chen Liang. --- .../hadoop/fs/permission/PermissionStatus.java | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/PermissionStatus.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/PermissionStatus.java index bc9e392a872..3c3693f613b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/PermissionStatus.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/PermissionStatus.java @@ -43,10 +43,6 @@ public class PermissionStatus implements Writable { public static PermissionStatus createImmutable( String user, String group, FsPermission permission) { return new PermissionStatus(user, group, permission) { - @Override - public PermissionStatus applyUMask(FsPermission umask) { - throw new UnsupportedOperationException(); - } @Override public void readFields(DataInput in) throws IOException { throw new UnsupportedOperationException(); @@ -76,15 +72,6 @@ public class PermissionStatus implements Writable { /** Return permission */ public FsPermission getPermission() {return permission;} - /** - * Apply umask. - * @see FsPermission#applyUMask(FsPermission) - */ - public PermissionStatus applyUMask(FsPermission umask) { - permission = permission.applyUMask(umask); - return this; - } - @Override public void readFields(DataInput in) throws IOException { username = Text.readString(in, Text.DEFAULT_MAX_LEN); From d4f73e7e27141ce0a88edb96fa304418a97a82a3 Mon Sep 17 00:00:00 2001 From: Ravi Prakash Date: Fri, 24 Mar 2017 09:38:17 -0700 Subject: [PATCH 120/188] HDFS-11574. Spelling mistakes in the Java source. Contributed by Hu Xiaodong. --- .../hdfs/server/namenode/ha/RequestHedgingProxyProvider.java | 2 +- .../java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java | 2 +- .../hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java | 2 +- .../hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java | 4 ++-- .../hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java | 2 +- .../hadoop/yarn/server/resourcemanager/Application.java | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java index a765e95411d..2f6c9bc7097 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java @@ -44,7 +44,7 @@ import org.slf4j.LoggerFactory; * per-se. It constructs a wrapper proxy that sends the request to ALL * underlying proxies simultaneously. It assumes the in an HA setup, there will * be only one Active, and the active should respond faster than any configured - * standbys. Once it recieve a response from any one of the configred proxies, + * standbys. Once it receive a response from any one of the configred proxies, * outstanding requests to other proxies are immediately cancelled. */ public class RequestHedgingProxyProvider extends diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java index aeff16da016..3f4fe288366 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java @@ -133,7 +133,7 @@ public class TestDataTransferProtocol { LOG.info("Expected: " + expected); if (eofExpected) { - throw new IOException("Did not recieve IOException when an exception " + + throw new IOException("Did not receive IOException when an exception " + "is expected while reading from " + datanode); } assertEquals(expected, received); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java index 12fa21103b3..e29d51851ea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java @@ -1282,7 +1282,7 @@ public class TestRetryCacheWithHA { /** * When NN failover happens, if the client did not receive the response and - * send a retry request to the other NN, the same response should be recieved + * send a retry request to the other NN, the same response should be received * based on the retry cache. */ public void testClientRetryWithFailover(final AtMostOnceOp op) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java index 7f74ed824c8..23395f8ab47 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java @@ -371,7 +371,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements "Message from ResourceManager: " + regNMResponse.getDiagnosticsMessage(); throw new YarnRuntimeException( - "Recieved SHUTDOWN signal from Resourcemanager, Registration of NodeManager failed, " + "Received SHUTDOWN signal from Resourcemanager, Registration of NodeManager failed, " + message); } @@ -965,7 +965,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements private boolean handleShutdownOrResyncCommand( NodeHeartbeatResponse response) { if (response.getNodeAction() == NodeAction.SHUTDOWN) { - LOG.warn("Recieved SHUTDOWN signal from Resourcemanager as part of" + LOG.warn("Received SHUTDOWN signal from Resourcemanager as part of" + " heartbeat, hence shutting down."); LOG.warn("Message from ResourceManager: " + response.getDiagnosticsMessage()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java index 0b93c724c44..ca029be95f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java @@ -1357,7 +1357,7 @@ public class TestNodeStatusUpdater { } }; verifyNodeStartFailure( - "Recieved SHUTDOWN signal from Resourcemanager, " + "Received SHUTDOWN signal from Resourcemanager, " + "Registration of NodeManager failed, " + "Message from ResourceManager: RM Shutting Down Node"); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java index da1bc2e21a7..e889de09f44 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java @@ -348,7 +348,7 @@ public class Application { if(LOG.isDebugEnabled()) { LOG.debug("getResources() for " + applicationId + ":" - + " ask=" + ask.size() + " recieved=" + containers.size()); + + " ask=" + ask.size() + " received=" + containers.size()); } return containers; From 0e556a5ba645570d381beca60114a1239b27d49f Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Fri, 24 Mar 2017 11:12:02 -0700 Subject: [PATCH 121/188] HADOOP-14211. FilterFs and ChRootedFs are too aggressive about enforcing 'authorityNeeded'. Contributed by Erik Krogen. --- .../src/main/java/org/apache/hadoop/fs/FilterFs.java | 3 +-- .../java/org/apache/hadoop/fs/viewfs/ChRootedFs.java | 3 +-- .../test/java/org/apache/hadoop/fs/TestFilterFs.java | 12 ++++++++++++ 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java index 6b1093e7969..5c16a4e9f36 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java @@ -57,8 +57,7 @@ public abstract class FilterFs extends AbstractFileSystem { } protected FilterFs(AbstractFileSystem fs) throws URISyntaxException { - super(fs.getUri(), fs.getUri().getScheme(), - fs.getUri().getAuthority() != null, fs.getUriDefaultPort()); + super(fs.getUri(), fs.getUri().getScheme(), false, fs.getUriDefaultPort()); myFs = fs; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java index d77ad8b96e3..ad1f5b56204 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java @@ -101,8 +101,7 @@ class ChRootedFs extends AbstractFileSystem { public ChRootedFs(final AbstractFileSystem fs, final Path theRoot) throws URISyntaxException { - super(fs.getUri(), fs.getUri().getScheme(), - fs.getUri().getAuthority() != null, fs.getUriDefaultPort()); + super(fs.getUri(), fs.getUri().getScheme(), false, fs.getUriDefaultPort()); myFs = fs; myFs.checkPath(theRoot); chRootPathPart = new Path(myFs.getUriPath(theRoot)); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFs.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFs.java index 27d093c2b0a..a2f09057ce6 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFs.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFs.java @@ -25,6 +25,8 @@ import java.util.Iterator; import junit.framework.TestCase; import org.apache.commons.logging.Log; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.viewfs.ConfigUtil; public class TestFilterFs extends TestCase { @@ -65,4 +67,14 @@ public class TestFilterFs extends TestCase { } } + // Test that FilterFs will accept an AbstractFileSystem to be filtered which + // has an optional authority, such as ViewFs + public void testFilteringWithNonrequiredAuthority() throws Exception { + Configuration conf = new Configuration(); + ConfigUtil.addLink(conf, "custom", "/mnt", URI.create("file:///")); + FileContext fc = + FileContext.getFileContext(URI.create("viewfs://custom/"), conf); + new FilterFs(fc.getDefaultFileSystem()) {}; + } + } From d49d1862a645b8c5d3eae95bf633bc82816f0168 Mon Sep 17 00:00:00 2001 From: Daniel Templeton Date: Fri, 24 Mar 2017 11:29:20 -0700 Subject: [PATCH 122/188] YARN-6360. Prevent FS state dump logger from cramming other log files (Contributed by Yufei Gu via Daniel Templeton) --- .../hadoop-common/src/main/conf/log4j.properties | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties index 60267635dfe..b4658ae7dae 100644 --- a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties +++ b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties @@ -324,6 +324,7 @@ log4j.appender.EWMA.maxUniqueMessages=${yarn.ewma.maxUniqueMessages} # Fair scheduler requests log on state dump log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.statedump=DEBUG,FSLOGGER +log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.statedump=false log4j.appender.FSLOGGER=org.apache.log4j.RollingFileAppender log4j.appender.FSLOGGER.File=${hadoop.log.dir}/fairscheduler-statedump.log log4j.appender.FSLOGGER.layout=org.apache.log4j.PatternLayout From 33815af4242ac8c6b119128730e63f13164fd763 Mon Sep 17 00:00:00 2001 From: Daniel Templeton Date: Fri, 24 Mar 2017 11:42:54 -0700 Subject: [PATCH 123/188] YARN-6334. TestRMFailover#testAutomaticFailover always passes even when it should fail (Contributed by Yufei Gu via Daniel Templeton) --- .../hadoop/yarn/client/TestRMFailover.java | 39 ++++++++++--------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java index 4bf6a781c91..d568d6a3eab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java @@ -30,6 +30,7 @@ import static org.mockito.Mockito.verify; import java.io.IOException; import java.net.HttpURLConnection; import java.net.URL; +import java.util.concurrent.TimeoutException; import javax.servlet.http.HttpServletResponse; @@ -40,6 +41,7 @@ import org.apache.hadoop.ha.ClientBaseWithFixes; import org.apache.hadoop.ha.HAServiceProtocol; import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; import org.apache.hadoop.service.Service.STATE; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.client.api.YarnClient; @@ -59,6 +61,8 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import com.google.common.base.Supplier; + public class TestRMFailover extends ClientBaseWithFixes { private static final Log LOG = LogFactory.getLog(TestRMFailover.class.getName()); @@ -159,6 +163,21 @@ public class TestRMFailover extends ClientBaseWithFixes { verifyConnections(); } + private void verifyRMTransitionToStandby(ResourceManager rm) + throws InterruptedException { + try { + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + return rm.getRMContext().getHAServiceState() == + HAServiceState.STANDBY; + } + }, 100, 20000); + } catch (TimeoutException e) { + fail("RM didn't transition to Standby."); + } + } + @Test public void testAutomaticFailover() throws YarnException, InterruptedException, IOException { @@ -182,15 +201,7 @@ public class TestRMFailover extends ClientBaseWithFixes { ResourceManager rm = cluster.getResourceManager( cluster.getActiveRMIndex()); rm.handleTransitionToStandByInNewThread(); - int maxWaitingAttempts = 2000; - while (maxWaitingAttempts-- > 0 ) { - if (rm.getRMContext().getHAServiceState() == HAServiceState.STANDBY) { - break; - } - Thread.sleep(1); - } - Assert.assertFalse("RM didn't transition to Standby ", - maxWaitingAttempts == 0); + verifyRMTransitionToStandby(rm); verifyConnections(); } @@ -393,15 +404,7 @@ public class TestRMFailover extends ClientBaseWithFixes { testThread.start(); testThread.join(); - int maxWaitingAttempts = 2000; - while (maxWaitingAttempts-- > 0) { - if (resourceManager.getRMContext().getHAServiceState() - == HAServiceState.STANDBY) { - break; - } - Thread.sleep(1); - } - assertFalse("RM didn't transition to Standby ", maxWaitingAttempts < 0); + verifyRMTransitionToStandby(resourceManager); } /** From 52b00600df921763725396ed92194d3338167655 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Fri, 24 Mar 2017 11:44:37 -0700 Subject: [PATCH 124/188] HADOOP-13715. Add isErasureCoded() API to FileStatus class. Contributed by Manoj Govindassamy. --- .../java/org/apache/hadoop/fs/FileStatus.java | 11 +++- .../hadoop/fs/permission/FsPermission.java | 7 +++ .../site/markdown/filesystem/filesystem.md | 9 +++ .../hadoop/fs/contract/ContractTestUtils.java | 27 +++++++++ .../fs/viewfs/TestViewfsFileStatus.java | 7 ++- .../hdfs/protocol/FsPermissionExtension.java | 17 +++++- .../hadoop/hdfs/web/JsonUtilClient.java | 13 ++-- .../fs/http/client/HttpFSFileSystem.java | 7 ++- .../hadoop/fs/http/server/FSOperations.java | 3 + .../fs/http/client/BaseTestHttpFSWith.java | 24 +++++++- .../apache/hadoop/test/TestHdfsHelper.java | 18 +++++- .../namenode/FSDirStatAndListingOp.java | 10 ++-- .../org/apache/hadoop/hdfs/web/JsonUtil.java | 6 ++ .../TestErasureCodingPolicyWithSnapshot.java | 26 ++++++++ .../apache/hadoop/hdfs/TestFileStatus.java | 15 +++-- .../hdfs/TestFileStatusWithECPolicy.java | 15 +++-- .../hdfs/server/namenode/FSAclBaseTest.java | 2 +- .../apache/hadoop/hdfs/web/TestWebHDFS.java | 60 +++++++++++++++++++ .../hadoop/fs/s3a/TestS3AGetFileStatus.java | 3 + .../hadoop/fs/adl/TestGetFileStatus.java | 6 +- 20 files changed, 256 insertions(+), 30 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java index 72ca24f12ce..26d39627f50 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java @@ -215,7 +215,16 @@ public class FileStatus implements Writable, Comparable, public boolean isEncrypted() { return permission.getEncryptedBit(); } - + + /** + * Tell whether the underlying file or directory is erasure coded or not. + * + * @return true if the underlying file or directory is erasure coded. + */ + public boolean isErasureCoded() { + return permission.getErasureCodedBit(); + } + /** * Get the owner of the file. * @return owner of the file. The string could be empty if there is no diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java index 56e19dcd777..ddb272498b5 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java @@ -312,6 +312,13 @@ public class FsPermission implements Writable, Serializable, return false; } + /** + * Returns true if the file or directory is erasure coded. + */ + public boolean getErasureCodedBit() { + return false; + } + /** Set the user file creation mask (umask) */ public static void setUMask(Configuration conf, FsPermission umask) { conf.set(UMASK_LABEL, String.format("%1$03o", umask.toShort())); diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md index 201d3974fd3..97bc7d19dfd 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md @@ -90,7 +90,16 @@ Get the status of a path stat.isEncrypted = True else stat.isEncrypted = False + if isErasureCoded(FS, p) : + stat.isErasureCoded = True + else + stat.isErasureCoded = False +The returned `FileStatus` status of the path additionally carries details on +Encryption and Erasure Coding information. `getFileStatus(Path p).isEncrypted()` +can be queried to find if the path is Encrypted. +Likewise, `getFileStatus(Path p).isErasureCoded()` will tell if the path is +Erasure Coded or not. ### `Path getHomeDirectory()` diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java index 90ae974506d..fd77045bdd2 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java @@ -519,6 +519,33 @@ public class ContractTestUtils extends Assert { fileStatus.isDirectory()); } + /** + * Assert that a path is Erasure Coded. + * + * @param fs filesystem + * @param path path of the file or directory + * @throws IOException on File IO problems + */ + public static void assertErasureCoded(final FileSystem fs, final Path path) + throws IOException { + FileStatus fileStatus = fs.getFileStatus(path); + assertTrue(path + " must be erasure coded!", fileStatus.isErasureCoded()); + } + + /** + * Assert that a path is not Erasure Coded. + * + * @param fs filesystem + * @param path path of the file or directory + * @throws IOException on File IO problems + */ + public static void assertNotErasureCoded(final FileSystem fs, + final Path path) throws IOException { + FileStatus fileStatus = fs.getFileStatus(path); + assertFalse(path + " should not be erasure coded!", + fileStatus.isErasureCoded()); + } + /** * Write the text to a file, returning the converted byte array * for use in validating the round trip. diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewfsFileStatus.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewfsFileStatus.java index e5489cb7151..449e2e33621 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewfsFileStatus.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewfsFileStatus.java @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.FsConstants; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.io.DataInputBuffer; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.test.GenericTestUtils; @@ -70,8 +71,11 @@ public class TestViewfsFileStatus { ConfigUtil.addLink(conf, "/foo/bar/baz", TEST_DIR.toURI()); FileSystem vfs = FileSystem.get(FsConstants.VIEWFS_URI, conf); assertEquals(ViewFileSystem.class, vfs.getClass()); - FileStatus stat = vfs.getFileStatus(new Path("/foo/bar/baz", testfilename)); + Path path = new Path("/foo/bar/baz", testfilename); + FileStatus stat = vfs.getFileStatus(path); assertEquals(content.length, stat.getLen()); + ContractTestUtils.assertNotErasureCoded(vfs, path); + // check serialization/deserialization DataOutputBuffer dob = new DataOutputBuffer(); stat.write(dob); @@ -80,6 +84,7 @@ public class TestViewfsFileStatus { FileStatus deSer = new FileStatus(); deSer.readFields(dib); assertEquals(content.length, deSer.getLen()); + assertFalse(deSer.isErasureCoded()); } // Tests that ViewFileSystem.getFileChecksum calls res.targetFileSystem diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/FsPermissionExtension.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/FsPermissionExtension.java index 786bb58e8eb..e0dd0d70467 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/FsPermissionExtension.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/FsPermissionExtension.java @@ -33,8 +33,10 @@ public class FsPermissionExtension extends FsPermission { private final static short ACL_BIT = 1 << 12; private final static short ENCRYPTED_BIT = 1 << 13; + private final static short ERASURE_CODED_BIT = 1 << 14; private final boolean aclBit; private final boolean encryptedBit; + private final boolean erasureCodedBit; /** * Constructs a new FsPermissionExtension based on the given FsPermission. @@ -42,10 +44,11 @@ public class FsPermissionExtension extends FsPermission { * @param perm FsPermission containing permission bits */ public FsPermissionExtension(FsPermission perm, boolean hasAcl, - boolean isEncrypted) { + boolean isEncrypted, boolean isErasureCoded) { super(perm.toShort()); aclBit = hasAcl; encryptedBit = isEncrypted; + erasureCodedBit = isErasureCoded; } /** @@ -57,12 +60,15 @@ public class FsPermissionExtension extends FsPermission { super(perm); aclBit = (perm & ACL_BIT) != 0; encryptedBit = (perm & ENCRYPTED_BIT) != 0; + erasureCodedBit = (perm & ERASURE_CODED_BIT) != 0; } @Override public short toExtendedShort() { - return (short)(toShort() | - (aclBit ? ACL_BIT : 0) | (encryptedBit ? ENCRYPTED_BIT : 0)); + return (short)(toShort() + | (aclBit ? ACL_BIT : 0) + | (encryptedBit ? ENCRYPTED_BIT : 0) + | (erasureCodedBit ? ERASURE_CODED_BIT : 0)); } @Override @@ -75,6 +81,11 @@ public class FsPermissionExtension extends FsPermission { return encryptedBit; } + @Override + public boolean getErasureCodedBit() { + return erasureCodedBit; + } + @Override public boolean equals(Object o) { // This intentionally delegates to the base class. This is only overridden diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java index cfcb4c9b07b..5e9396e368d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java @@ -98,12 +98,13 @@ class JsonUtilClient { /** Convert a string to a FsPermission object. */ static FsPermission toFsPermission( - final String s, Boolean aclBit, Boolean encBit) { + final String s, Boolean aclBit, Boolean encBit, Boolean erasureBit) { FsPermission perm = new FsPermission(Short.parseShort(s, 8)); final boolean aBit = (aclBit != null) ? aclBit : false; final boolean eBit = (encBit != null) ? encBit : false; - if (aBit || eBit) { - return new FsPermissionExtension(perm, aBit, eBit); + final boolean ecBit = (erasureBit != null) ? erasureBit : false; + if (aBit || eBit || ecBit) { + return new FsPermissionExtension(perm, aBit, eBit, ecBit); } else { return perm; } @@ -129,7 +130,8 @@ class JsonUtilClient { final String group = (String) m.get("group"); final FsPermission permission = toFsPermission((String) m.get("permission"), (Boolean) m.get("aclBit"), - (Boolean) m.get("encBit")); + (Boolean) m.get("encBit"), + (Boolean) m.get("ecBit")); final long aTime = ((Number) m.get("accessTime")).longValue(); final long mTime = ((Number) m.get("modificationTime")).longValue(); final long blockSize = ((Number) m.get("blockSize")).longValue(); @@ -464,7 +466,8 @@ class JsonUtilClient { String permString = (String) m.get("permission"); if (permString != null) { final FsPermission permission = toFsPermission(permString, - (Boolean) m.get("aclBit"), (Boolean) m.get("encBit")); + (Boolean) m.get("aclBit"), (Boolean) m.get("encBit"), + (Boolean) m.get("ecBit")); aclStatusBuilder.setPermission(permission); } final List entries = (List) m.get("entries"); diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java index dad8df21d6a..055a57ee82a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java @@ -193,6 +193,7 @@ public class HttpFSFileSystem extends FileSystem public static final String ACL_BIT_JSON = "aclBit"; public static final String ENC_BIT_JSON = "encBit"; + public static final String EC_BIT_JSON = "ecBit"; public static final String DIRECTORY_LISTING_JSON = "DirectoryListing"; public static final String PARTIAL_LISTING_JSON = "partialListing"; @@ -1042,11 +1043,13 @@ public class HttpFSFileSystem extends FileSystem final String s = (String) json.get(PERMISSION_JSON); final Boolean aclBit = (Boolean) json.get(ACL_BIT_JSON); final Boolean encBit = (Boolean) json.get(ENC_BIT_JSON); + final Boolean erasureBit = (Boolean) json.get(EC_BIT_JSON); FsPermission perm = new FsPermission(Short.parseShort(s, 8)); final boolean aBit = (aclBit != null) ? aclBit : false; final boolean eBit = (encBit != null) ? encBit : false; - if (aBit || eBit) { - return new FsPermissionExtension(perm, aBit, eBit); + final boolean ecBit = (erasureBit != null) ? erasureBit : false; + if (aBit || eBit || ecBit) { + return new FsPermissionExtension(perm, aBit, eBit, ecBit); } else { return perm; } diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java index 6de701232f4..eec6981da02 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java @@ -114,6 +114,9 @@ public class FSOperations { if (fileStatus.getPermission().getEncryptedBit()) { json.put(HttpFSFileSystem.ENC_BIT_JSON, true); } + if (fileStatus.getPermission().getErasureCodedBit()) { + json.put(HttpFSFileSystem.EC_BIT_JSON, true); + } return json; } diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java index 2d86794de9b..36d0ad98abc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystemTestHelper; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.fs.http.server.HttpFSServerWebApp; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; @@ -944,6 +945,24 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase { assertFalse(httpStatus.isEncrypted()); } + private void testErasureCoding() throws Exception { + Assume.assumeFalse("Assume its not a local FS!", isLocalFS()); + FileSystem proxyFs = FileSystem.get(getProxiedFSConf()); + FileSystem httpFS = getHttpFSFileSystem(); + Path filePath = new Path(getProxiedFSTestDir(), "foo.txt"); + proxyFs.create(filePath).close(); + + ContractTestUtils.assertNotErasureCoded(httpFS, getProxiedFSTestDir()); + ContractTestUtils.assertNotErasureCoded(httpFS, filePath); + ContractTestUtils.assertErasureCoded(httpFS, + TestHdfsHelper.ERASURE_CODING_DIR); + ContractTestUtils.assertErasureCoded(httpFS, + TestHdfsHelper.ERASURE_CODING_FILE); + + proxyFs.close(); + httpFS.close(); + } + private void testStoragePolicy() throws Exception { Assume.assumeFalse("Assume its not a local FS", isLocalFS()); FileSystem fs = FileSystem.get(getProxiedFSConf()); @@ -993,7 +1012,7 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase { WORKING_DIRECTORY, MKDIRS, SET_TIMES, SET_PERMISSION, SET_OWNER, SET_REPLICATION, CHECKSUM, CONTENT_SUMMARY, FILEACLS, DIRACLS, SET_XATTR, GET_XATTRS, REMOVE_XATTR, LIST_XATTRS, ENCRYPTION, LIST_STATUS_BATCH, - GETTRASHROOT, STORAGEPOLICY + GETTRASHROOT, STORAGEPOLICY, ERASURE_CODING } private void operation(Operation op) throws Exception { @@ -1079,6 +1098,9 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase { case STORAGEPOLICY: testStoragePolicy(); break; + case ERASURE_CODING: + testErasureCoding(); + break; } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestHdfsHelper.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestHdfsHelper.java index 0e701f712e7..258dde54f82 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestHdfsHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestHdfsHelper.java @@ -31,6 +31,9 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.junit.Test; import org.junit.runners.model.FrameworkMethod; import org.junit.runners.model.Statement; @@ -136,6 +139,10 @@ public class TestHdfsHelper extends TestDirHelper { public static final Path ENCRYPTION_ZONE = new Path("/ez"); public static final Path ENCRYPTED_FILE = new Path("/ez/encfile"); + public static final Path ERASURE_CODING_DIR = new Path("/ec"); + public static final Path ERASURE_CODING_FILE = new Path("/ec/ecfile"); + public static final ErasureCodingPolicy ERASURE_CODING_POLICY = + ErasureCodingPolicyManager.getPolicyByID(HdfsConstants.XOR_2_1_POLICY_ID); private static MiniDFSCluster MINI_DFS = null; @@ -161,8 +168,12 @@ public class TestHdfsHelper extends TestDirHelper { new Path(helper.getTestRootDir(), "test.jks").toUri(); conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, jceksPath); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + ERASURE_CODING_POLICY.getName()); MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf); - builder.numDataNodes(2); + int totalDataNodes = ERASURE_CODING_POLICY.getNumDataUnits() + + ERASURE_CODING_POLICY.getNumParityUnits(); + builder.numDataNodes(totalDataNodes); MiniDFSCluster miniHdfs = builder.build(); final String testkey = "testkey"; DFSTestUtil.createKey(testkey, miniHdfs, conf); @@ -179,6 +190,11 @@ public class TestHdfsHelper extends TestDirHelper { fileSystem.createEncryptionZone(ENCRYPTION_ZONE, testkey); fileSystem.create(ENCRYPTED_FILE).close(); + fileSystem.mkdirs(ERASURE_CODING_DIR); + fileSystem.setErasureCodingPolicy(ERASURE_CODING_DIR, + ERASURE_CODING_POLICY.getName()); + fileSystem.create(ERASURE_CODING_FILE).close(); + MINI_DFS = miniHdfs; } return MINI_DFS; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java index d0431ff1929..04efa652d8d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java @@ -414,6 +414,7 @@ class FSDirStatAndListingOp { final ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp .unprotectedGetErasureCodingPolicy(fsd.getFSNamesystem(), iip); + final boolean isErasureCoded = (ecPolicy != null); if (node.isFile()) { final INodeFile fileNode = node.asFile(); @@ -448,7 +449,7 @@ class FSDirStatAndListingOp { blocksize, node.getModificationTime(snapshot), node.getAccessTime(snapshot), - getPermissionForFileStatus(nodeAttrs, isEncrypted), + getPermissionForFileStatus(nodeAttrs, isEncrypted, isErasureCoded), nodeAttrs.getUserName(), nodeAttrs.getGroupName(), node.isSymlink() ? node.asSymlink().getSymlink() : null, @@ -489,11 +490,12 @@ class FSDirStatAndListingOp { * and encrypted bit on if it represents an encrypted file/dir. */ private static FsPermission getPermissionForFileStatus( - INodeAttributes node, boolean isEncrypted) { + INodeAttributes node, boolean isEncrypted, boolean isErasureCoded) { FsPermission perm = node.getFsPermission(); boolean hasAcl = node.getAclFeature() != null; - if (hasAcl || isEncrypted) { - perm = new FsPermissionExtension(perm, hasAcl, isEncrypted); + if (hasAcl || isEncrypted || isErasureCoded) { + perm = new FsPermissionExtension(perm, hasAcl, + isEncrypted, isErasureCoded); } return perm; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java index 827e1a320ff..a8861a8ccb8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java @@ -124,6 +124,9 @@ public class JsonUtil { if (perm.getEncryptedBit()) { m.put("encBit", true); } + if (perm.getErasureCodedBit()) { + m.put("ecBit", true); + } m.put("accessTime", status.getAccessTime()); m.put("modificationTime", status.getModificationTime()); m.put("blockSize", status.getBlockSize()); @@ -376,6 +379,9 @@ public class JsonUtil { if (perm.getEncryptedBit()) { m.put("encBit", true); } + if (perm.getErasureCodedBit()) { + m.put("ecBit", true); + } } final Map> finalMap = new TreeMap>(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java index c5651ee89ef..5549bf99400 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java @@ -25,6 +25,7 @@ import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FsShell; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; @@ -199,4 +200,29 @@ public class TestErasureCodingPolicyWithSnapshot { assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy, fs.getErasureCodingPolicy(snap1)); } + + @Test (timeout = 300000) + public void testFileStatusAcrossNNRestart() throws IOException { + final int len = 1024; + final Path normalFile = new Path("/", "normalFile"); + DFSTestUtil.createFile(fs, normalFile, len, (short) 1, 0xFEED); + + final Path ecDir = new Path("/ecdir"); + final Path ecFile = new Path(ecDir, "ecFile"); + fs.mkdirs(ecDir); + + // Set erasure coding policy + fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName()); + DFSTestUtil.createFile(fs, ecFile, len, (short) 1, 0xFEED); + + // Verify FileStatus for normal and EC files + ContractTestUtils.assertNotErasureCoded(fs, normalFile); + ContractTestUtils.assertErasureCoded(fs, ecFile); + + cluster.restartNameNode(true); + + // Verify FileStatus for normal and EC files + ContractTestUtils.assertNotErasureCoded(fs, normalFile); + ContractTestUtils.assertErasureCoded(fs, ecFile); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java index f1158599cfe..7631e7d70cc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java @@ -33,6 +33,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.ipc.RemoteException; @@ -97,7 +98,8 @@ public class TestFileStatus { Path path = new Path("/"); assertTrue("/ should be a directory", fs.getFileStatus(path).isDirectory()); - + ContractTestUtils.assertNotErasureCoded(fs, path); + // Make sure getFileInfo returns null for files which do not exist HdfsFileStatus fileInfo = dfsClient.getFileInfo("/noSuchFile"); assertEquals("Non-existant file should result in null", null, fileInfo); @@ -133,7 +135,8 @@ public class TestFileStatus { assertEquals(blockSize, status.getBlockSize()); assertEquals(1, status.getReplication()); assertEquals(fileSize, status.getLen()); - assertEquals(file1.makeQualified(fs.getUri(), + ContractTestUtils.assertNotErasureCoded(fs, file1); + assertEquals(file1.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString(), status.getPath().toString()); } @@ -148,10 +151,11 @@ public class TestFileStatus { assertEquals(blockSize, status.getBlockSize()); assertEquals(1, status.getReplication()); assertEquals(fileSize, status.getLen()); - assertEquals(file1.makeQualified(fs.getUri(), + ContractTestUtils.assertNotErasureCoded(fs, file1); + assertEquals(file1.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString(), status.getPath().toString()); - + RemoteIterator itor = fc.listStatus(file1); status = itor.next(); assertEquals(stats[0], status); @@ -196,7 +200,8 @@ public class TestFileStatus { FileStatus status = fs.getFileStatus(dir); assertTrue(dir + " should be a directory", status.isDirectory()); assertTrue(dir + " should be zero size ", status.getLen() == 0); - assertEquals(dir.makeQualified(fs.getUri(), + ContractTestUtils.assertNotErasureCoded(fs, dir); + assertEquals(dir.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString(), status.getPath().toString()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java index f033ab75b91..0e270ff75c7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java @@ -23,8 +23,8 @@ import static org.junit.Assert.assertTrue; import java.io.IOException; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.junit.After; @@ -66,25 +66,32 @@ public class TestFileStatusWithECPolicy { // test directory doesn't have an EC policy final Path dir = new Path("/foo"); assertTrue(fs.mkdir(dir, FsPermission.getDirDefault())); + ContractTestUtils.assertNotErasureCoded(fs, dir); assertNull(client.getFileInfo(dir.toString()).getErasureCodingPolicy()); // test file doesn't have an EC policy final Path file = new Path(dir, "foo"); fs.create(file).close(); assertNull(client.getFileInfo(file.toString()).getErasureCodingPolicy()); + ContractTestUtils.assertNotErasureCoded(fs, file); fs.delete(file, true); - final ErasureCodingPolicy ecPolicy1 = StripedFileTestUtil.getDefaultECPolicy(); + final ErasureCodingPolicy ecPolicy1 = + StripedFileTestUtil.getDefaultECPolicy(); // set EC policy on dir fs.setErasureCodingPolicy(dir, ecPolicy1.getName()); - final ErasureCodingPolicy ecPolicy2 = client.getFileInfo(dir.toUri().getPath()).getErasureCodingPolicy(); + ContractTestUtils.assertErasureCoded(fs, dir); + final ErasureCodingPolicy ecPolicy2 = + client.getFileInfo(dir.toUri().getPath()).getErasureCodingPolicy(); assertNotNull(ecPolicy2); assertTrue(ecPolicy1.equals(ecPolicy2)); // test file doesn't have an EC policy fs.create(file).close(); final ErasureCodingPolicy ecPolicy3 = - fs.getClient().getFileInfo(file.toUri().getPath()).getErasureCodingPolicy(); + fs.getClient().getFileInfo(file.toUri().getPath()) + .getErasureCodingPolicy(); assertNotNull(ecPolicy3); assertTrue(ecPolicy1.equals(ecPolicy3)); + ContractTestUtils.assertErasureCoded(fs, file); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java index 9f5528eab5f..81270e6f2bd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java @@ -862,7 +862,7 @@ public abstract class FSAclBaseTest { assertPermission((short)0700); fs.setPermission(path, new FsPermissionExtension(FsPermission. - createImmutable((short)0755), true, true)); + createImmutable((short)0755), true, true, true)); INode inode = cluster.getNamesystem().getFSDirectory() .getINode(path.toUri().getPath(), DirOp.READ_LINK); assertNotNull(inode); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java index 7cfaa99294b..3c366c39dad 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java @@ -65,6 +65,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclEntryScope; import org.apache.hadoop.fs.permission.AclEntryType; @@ -85,6 +86,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; +import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper; import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods; @@ -513,6 +515,64 @@ public class TestWebHDFS { } } + @Test (timeout = 60000) + public void testWebHdfsErasureCodingFiles() throws Exception { + MiniDFSCluster cluster = null; + final Configuration conf = WebHdfsTestUtil.createConf(); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + ErasureCodingPolicyManager.getPolicyByID( + HdfsConstants.XOR_2_1_POLICY_ID).getName()); + try { + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build(); + cluster.waitActive(); + final DistributedFileSystem dfs = cluster.getFileSystem(); + final WebHdfsFileSystem webHdfs = WebHdfsTestUtil + .getWebHdfsFileSystem(conf, WebHdfsConstants.WEBHDFS_SCHEME); + + final Path ecDir = new Path("/ec"); + dfs.mkdirs(ecDir); + dfs.setErasureCodingPolicy(ecDir, + ErasureCodingPolicyManager.getPolicyByID( + HdfsConstants.XOR_2_1_POLICY_ID).getName()); + final Path ecFile = new Path(ecDir, "ec-file.log"); + DFSTestUtil.createFile(dfs, ecFile, 1024 * 10, (short) 1, 0xFEED); + + final Path normalDir = new Path("/dir"); + dfs.mkdirs(normalDir); + final Path normalFile = new Path(normalDir, "file.log"); + DFSTestUtil.createFile(dfs, normalFile, 1024 * 10, (short) 1, 0xFEED); + + FileStatus expectedECDirStatus = dfs.getFileStatus(ecDir); + FileStatus actualECDirStatus = webHdfs.getFileStatus(ecDir); + Assert.assertEquals(expectedECDirStatus.isErasureCoded(), + actualECDirStatus.isErasureCoded()); + ContractTestUtils.assertErasureCoded(dfs, ecDir); + + FileStatus expectedECFileStatus = dfs.getFileStatus(ecFile); + FileStatus actualECFileStatus = webHdfs.getFileStatus(ecFile); + Assert.assertEquals(expectedECFileStatus.isErasureCoded(), + actualECFileStatus.isErasureCoded()); + ContractTestUtils.assertErasureCoded(dfs, ecFile); + + FileStatus expectedNormalDirStatus = dfs.getFileStatus(normalDir); + FileStatus actualNormalDirStatus = webHdfs.getFileStatus(normalDir); + Assert.assertEquals(expectedNormalDirStatus.isErasureCoded(), + actualNormalDirStatus.isErasureCoded()); + ContractTestUtils.assertNotErasureCoded(dfs, normalDir); + + FileStatus expectedNormalFileStatus = dfs.getFileStatus(normalFile); + FileStatus actualNormalFileStatus = webHdfs.getFileStatus(normalDir); + Assert.assertEquals(expectedNormalFileStatus.isErasureCoded(), + actualNormalFileStatus.isErasureCoded()); + ContractTestUtils.assertNotErasureCoded(dfs, normalFile); + + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } + /** * Test snapshot creation through WebHdfs */ diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java index a5dc01ab87a..deb3963b2d2 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java @@ -35,6 +35,7 @@ import com.amazonaws.services.s3.model.S3ObjectSummary; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.contract.ContractTestUtils; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; import org.hamcrest.Matcher; @@ -60,6 +61,7 @@ public class TestS3AGetFileStatus extends AbstractS3AMockTest { assertTrue(stat.isFile()); assertEquals(meta.getContentLength(), stat.getLen()); assertEquals(meta.getLastModified().getTime(), stat.getModificationTime()); + ContractTestUtils.assertNotErasureCoded(fs, path); } @Test @@ -98,6 +100,7 @@ public class TestS3AGetFileStatus extends AbstractS3AMockTest { assertNotNull(stat); assertEquals(fs.makeQualified(path), stat.getPath()); assertTrue(stat.isDirectory()); + ContractTestUtils.assertNotErasureCoded(fs, path); } @Test diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java index 78ef9313bac..df39a40cebd 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java @@ -51,8 +51,8 @@ public class TestGetFileStatus extends AdlMockWebServer { getMockServer().enqueue(new MockResponse().setResponseCode(200) .setBody(TestADLResponseData.getGetFileStatusJSONResponse())); long startTime = Time.monotonicNow(); - FileStatus fileStatus = getMockAdlFileSystem() - .getFileStatus(new Path("/test1/test2")); + Path path = new Path("/test1/test2"); + FileStatus fileStatus = getMockAdlFileSystem().getFileStatus(path); long endTime = Time.monotonicNow(); LOG.debug("Time : " + (endTime - startTime)); Assert.assertTrue(fileStatus.isFile()); @@ -65,6 +65,8 @@ public class TestGetFileStatus extends AdlMockWebServer { Assert.assertEquals(new FsPermission("777"), fileStatus.getPermission()); Assert.assertEquals("NotSupportYet", fileStatus.getOwner()); Assert.assertEquals("NotSupportYet", fileStatus.getGroup()); + Assert.assertFalse(path + " should not be erasure coded!", + fileStatus.isErasureCoded()); } @Test From 332a997e10cca88d9ab3aa8252102366b628eaec Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Fri, 24 Mar 2017 12:56:46 -0700 Subject: [PATCH 125/188] HDFS-11170. Add builder-based create API to FileSystem. Contributed by SammiChen and Wei Zhou. --- .../hadoop/fs/FSDataOutputStreamBuilder.java | 142 ++++++++++++++++++ .../java/org/apache/hadoop/fs/FileSystem.java | 9 ++ .../apache/hadoop/fs/FilterFileSystem.java | 5 + .../org/apache/hadoop/fs/HarFileSystem.java | 5 + .../apache/hadoop/fs/TestLocalFileSystem.java | 54 +++++++ .../hadoop/hdfs/DistributedFileSystem.java | 81 ++++++++++ .../hdfs/TestDistributedFileSystem.java | 35 ++++- .../namenode/TestFavoredNodesEndToEnd.java | 23 +++ 8 files changed, 353 insertions(+), 1 deletion(-) create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java new file mode 100644 index 00000000000..2e885f3460c --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.fs.Options.ChecksumOpt; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.util.Progressable; + +import java.io.IOException; +import java.util.EnumSet; + +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY; + +/** Base of specific file system FSDataOutputStreamBuilder. */ +public class FSDataOutputStreamBuilder{ + private Path path = null; + private FsPermission permission = null; + private Integer bufferSize; + private Short replication; + private Long blockSize; + private Progressable progress = null; + private EnumSet flags = null; + private ChecksumOpt checksumOpt = null; + + private final FileSystem fs; + + public FSDataOutputStreamBuilder(FileSystem fileSystem, Path p) { + fs = fileSystem; + path = p; + } + + protected Path getPath() { + return path; + } + + protected FsPermission getPermission() { + if (permission == null) { + return FsPermission.getFileDefault(); + } + return permission; + } + + public FSDataOutputStreamBuilder setPermission(final FsPermission perm) { + Preconditions.checkNotNull(perm); + permission = perm; + return this; + } + + protected int getBufferSize() { + if (bufferSize == null) { + return fs.getConf().getInt(IO_FILE_BUFFER_SIZE_KEY, + IO_FILE_BUFFER_SIZE_DEFAULT); + } + return bufferSize; + } + + public FSDataOutputStreamBuilder setBufferSize(int bufSize) { + bufferSize = bufSize; + return this; + } + + protected short getReplication() { + if (replication == null) { + return fs.getDefaultReplication(getPath()); + } + return replication; + } + + public FSDataOutputStreamBuilder setReplication(short replica) { + replication = replica; + return this; + } + + protected long getBlockSize() { + if (blockSize == null) { + return fs.getDefaultBlockSize(getPath()); + } + return blockSize; + } + + public FSDataOutputStreamBuilder setBlockSize(long blkSize) { + blockSize = blkSize; + return this; + } + + protected Progressable getProgress() { + return progress; + } + + public FSDataOutputStreamBuilder setProgress(final Progressable prog) { + Preconditions.checkNotNull(prog); + progress = prog; + return this; + } + + protected EnumSet getFlags() { + if (flags == null) { + return EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE); + } + return flags; + } + + public FSDataOutputStreamBuilder setFlags( + final EnumSet enumFlags) { + Preconditions.checkNotNull(enumFlags); + flags = enumFlags; + return this; + } + + protected ChecksumOpt getChecksumOpt() { + return checksumOpt; + } + + public FSDataOutputStreamBuilder setChecksumOpt( + final ChecksumOpt chksumOpt) { + Preconditions.checkNotNull(chksumOpt); + checksumOpt = chksumOpt; + return this; + } + + public FSDataOutputStream build() throws IOException { + return fs.create(getPath(), getPermission(), getFlags(), getBufferSize(), + getReplication(), getBlockSize(), getProgress(), getChecksumOpt()); + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index ededfa9810f..c3282e5396e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -4138,4 +4138,13 @@ public abstract class FileSystem extends Configured implements Closeable { public static GlobalStorageStatistics getGlobalStorageStatistics() { return GlobalStorageStatistics.INSTANCE; } + + /** + * Create a new FSDataOutputStreamBuilder for the file with path. + * @param path file path + * @return a FSDataOutputStreamBuilder object to build the file + */ + public FSDataOutputStreamBuilder newFSDataOutputStreamBuilder(Path path) { + return new FSDataOutputStreamBuilder(this, path); + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java index 41429ace83f..ef0945896ef 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java @@ -665,4 +665,9 @@ public class FilterFileSystem extends FileSystem { public Collection getTrashRoots(boolean allUsers) { return fs.getTrashRoots(allUsers); } + + @Override + public FSDataOutputStreamBuilder newFSDataOutputStreamBuilder(Path path) { + return fs.newFSDataOutputStreamBuilder(path); + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java index ce1cf4556e4..7e50ab1c674 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java @@ -1268,4 +1268,9 @@ public class HarFileSystem extends FileSystem { public short getDefaultReplication(Path f) { return fs.getDefaultReplication(f); } + + @Override + public FSDataOutputStreamBuilder newFSDataOutputStreamBuilder(Path path) { + return fs.newFSDataOutputStreamBuilder(path); + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java index 23113375e72..5da5a4a97da 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java @@ -19,10 +19,13 @@ package org.apache.hadoop.fs; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem.Statistics; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.StringUtils; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY; import static org.apache.hadoop.fs.FileSystemTestHelper.*; import java.io.*; @@ -636,4 +639,55 @@ public class TestLocalFileSystem { FileStatus[] stats = fs.listStatus(path); assertTrue(stats != null && stats.length == 1 && stats[0] == stat); } + + @Test + public void testFSOutputStreamBuilder() throws Exception { + Path path = new Path(TEST_ROOT_DIR, "testBuilder"); + + try { + FSDataOutputStreamBuilder builder = + fileSys.newFSDataOutputStreamBuilder(path); + FSDataOutputStream out = builder.build(); + String content = "Create with a generic type of createBuilder!"; + byte[] contentOrigin = content.getBytes("UTF8"); + out.write(contentOrigin); + out.close(); + + FSDataInputStream input = fileSys.open(path); + byte[] buffer = + new byte[(int) (fileSys.getFileStatus(path).getLen())]; + input.readFully(0, buffer); + input.close(); + Assert.assertArrayEquals("The data be read should equals with the " + + "data written.", contentOrigin, buffer); + } catch (IOException e) { + throw e; + } + + // Test value not being set for replication, block size, buffer size + // and permission + FSDataOutputStreamBuilder builder = + fileSys.newFSDataOutputStreamBuilder(path); + builder.build(); + Assert.assertEquals("Should be default block size", + builder.getBlockSize(), fileSys.getDefaultBlockSize()); + Assert.assertEquals("Should be default replication factor", + builder.getReplication(), fileSys.getDefaultReplication()); + Assert.assertEquals("Should be default buffer size", + builder.getBufferSize(), + fileSys.getConf().getInt(IO_FILE_BUFFER_SIZE_KEY, + IO_FILE_BUFFER_SIZE_DEFAULT)); + Assert.assertEquals("Should be default permission", + builder.getPermission(), FsPermission.getFileDefault()); + + // Test set 0 to replication, block size and buffer size + builder = fileSys.newFSDataOutputStreamBuilder(path); + builder.setBufferSize(0).setBlockSize(0).setReplication((short) 0); + Assert.assertEquals("Block size should be 0", + builder.getBlockSize(), 0); + Assert.assertEquals("Replication factor should be 0", + builder.getReplication(), 0); + Assert.assertEquals("Buffer size should be 0", + builder.getBufferSize(), 0); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 1eef560ec57..5cd956cf346 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -39,6 +39,7 @@ import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FSDataOutputStreamBuilder; import org.apache.hadoop.fs.FSLinkResolver; import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileEncryptionInfo; @@ -446,6 +447,48 @@ public class DistributedFileSystem extends FileSystem { }.resolve(this, absF); } + /** + * Same as + * {@link #create(Path, FsPermission, EnumSet, int, short, long, + * Progressable, ChecksumOpt)} with the addition of favoredNodes that is a + * hint to where the namenode should place the file blocks. + * The favored nodes hint is not persisted in HDFS. Hence it may be honored + * at the creation time only. And with favored nodes, blocks will be pinned + * on the datanodes to prevent balancing move the block. HDFS could move the + * blocks during replication, to move the blocks from favored nodes. A value + * of null means no favored nodes for this create + */ + private HdfsDataOutputStream create(final Path f, + final FsPermission permission, EnumSet flag, + final int bufferSize, final short replication, final long blockSize, + final Progressable progress, final ChecksumOpt checksumOpt, + final InetSocketAddress[] favoredNodes) throws IOException { + statistics.incrementWriteOps(1); + storageStatistics.incrementOpCounter(OpType.CREATE); + Path absF = fixRelativePart(f); + return new FileSystemLinkResolver() { + @Override + public HdfsDataOutputStream doCall(final Path p) throws IOException { + final DFSOutputStream out = dfs.create(getPathName(f), permission, + flag, true, replication, blockSize, progress, bufferSize, + checksumOpt, favoredNodes); + return dfs.createWrappedOutputStream(out, statistics); + } + @Override + public HdfsDataOutputStream next(final FileSystem fs, final Path p) + throws IOException { + if (fs instanceof DistributedFileSystem) { + DistributedFileSystem myDfs = (DistributedFileSystem)fs; + return myDfs.create(p, permission, flag, bufferSize, replication, + blockSize, progress, checksumOpt, favoredNodes); + } + throw new UnsupportedOperationException("Cannot create with" + + " favoredNodes through a symlink to a non-DistributedFileSystem: " + + f + " -> " + p); + } + }.resolve(this, absF); + } + @Override protected HdfsDataOutputStream primitiveCreate(Path f, FsPermission absolutePermission, EnumSet flag, int bufferSize, @@ -2584,4 +2627,42 @@ public class DistributedFileSystem extends FileSystem { DFSOpsCountStatistics getDFSOpsCountStatistics() { return storageStatistics; } + + /** + * Extends FSDataOutputStreamBuilder to support special requirements + * of DistributedFileSystem. + */ + public static class HdfsDataOutputStreamBuilder + extends FSDataOutputStreamBuilder { + private final DistributedFileSystem dfs; + private InetSocketAddress[] favoredNodes = null; + + public HdfsDataOutputStreamBuilder(DistributedFileSystem dfs, Path path) { + super(dfs, path); + this.dfs = dfs; + } + + protected InetSocketAddress[] getFavoredNodes() { + return favoredNodes; + } + + public HdfsDataOutputStreamBuilder setFavoredNodes( + final InetSocketAddress[] nodes) { + Preconditions.checkNotNull(nodes); + favoredNodes = nodes.clone(); + return this; + } + + @Override + public HdfsDataOutputStream build() throws IOException { + return dfs.create(getPath(), getPermission(), getFlags(), + getBufferSize(), getReplication(), getBlockSize(), + getProgress(), getChecksumOpt(), getFavoredNodes()); + } + } + + @Override + public HdfsDataOutputStreamBuilder newFSDataOutputStreamBuilder(Path path) { + return new HdfsDataOutputStreamBuilder(this, path); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java index 76545316b6b..e9af5949214 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java @@ -69,6 +69,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.StorageStatistics.LongStatistic; import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.hdfs.client.impl.LeaseRenewer; @@ -81,7 +82,6 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; -import org.apache.hadoop.hdfs.server.namenode.top.window.RollingWindowManager.Op; import org.apache.hadoop.hdfs.web.WebHdfsConstants; import org.apache.hadoop.net.DNSToSwitchMapping; import org.apache.hadoop.net.NetUtils; @@ -1410,4 +1410,37 @@ public class TestDistributedFileSystem { } } } + + @Test + public void testDFSDataOutputStreamBuilder() throws Exception { + Configuration conf = getTestConfiguration(); + MiniDFSCluster cluster = null; + String testFile = "/testDFSDataOutputStreamBuilder"; + Path testFilePath = new Path(testFile); + try { + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + DistributedFileSystem fs = cluster.getFileSystem(); + + // Test create an empty file + FSDataOutputStream out = + fs.newFSDataOutputStreamBuilder(testFilePath).build(); + out.close(); + + // Test create a file with content, and verify the content + String content = "This is a test!"; + out = fs.newFSDataOutputStreamBuilder(testFilePath) + .setBufferSize(4096).setReplication((short) 1) + .setBlockSize(4096).build(); + byte[] contentOrigin = content.getBytes("UTF8"); + out.write(contentOrigin); + out.close(); + + ContractTestUtils.verifyFileContents(fs, testFilePath, + content.getBytes()); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFavoredNodesEndToEnd.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFavoredNodesEndToEnd.java index b78b6cc36ed..50e56cc7e81 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFavoredNodesEndToEnd.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFavoredNodesEndToEnd.java @@ -189,6 +189,29 @@ public class TestFavoredNodesEndToEnd { } } + @Test(timeout = 180000) + public void testCreateStreamBuilderFavoredNodesEndToEnd() throws Exception { + //create 10 files with random preferred nodes + for (int i = 0; i < NUM_FILES; i++) { + Random rand = new Random(System.currentTimeMillis() + i); + //pass a new created rand so as to get a uniform distribution each time + //without too much collisions (look at the do-while loop in getDatanodes) + InetSocketAddress[] dns = getDatanodes(rand); + Path p = new Path("/filename"+i); + FSDataOutputStream out = + dfs.newFSDataOutputStreamBuilder(p).setFavoredNodes(dns).build(); + out.write(SOME_BYTES); + out.close(); + BlockLocation[] locations = getBlockLocations(p); + //verify the files got created in the right nodes + for (BlockLocation loc : locations) { + String[] hosts = loc.getNames(); + String[] hosts1 = getStringForInetSocketAddrs(dns); + assertTrue(compareNodes(hosts, hosts1)); + } + } + } + private BlockLocation[] getBlockLocations(Path p) throws Exception { DFSTestUtil.waitReplication(dfs, p, (short)3); BlockLocation[] locations = dfs.getClient().getBlockLocations( From d1b7439b48caa18d64a94be1ad5e4927ce573ab8 Mon Sep 17 00:00:00 2001 From: John Zhuge Date: Fri, 24 Mar 2017 08:36:34 -0700 Subject: [PATCH 126/188] HADOOP-14230. TestAdlFileSystemContractLive fails to clean up. Contributed by John Zhuge. --- .../hadoop/fs/adl/live/TestAdlFileSystemContractLive.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlFileSystemContractLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlFileSystemContractLive.java index 657947e7b0f..9d055f18b2d 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlFileSystemContractLive.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlFileSystemContractLive.java @@ -45,9 +45,8 @@ public class TestAdlFileSystemContractLive extends FileSystemContractBaseTest { protected void tearDown() throws Exception { if (AdlStorageConfiguration.isContractTestEnabled()) { cleanup(); - adlStore = null; - fs = null; } + super.tearDown(); } private void cleanup() throws IOException { From 1168ece59640d8ad3166e355d2e82deec2fbaf14 Mon Sep 17 00:00:00 2001 From: Xiaobing Zhou Date: Fri, 24 Mar 2017 14:33:44 -0700 Subject: [PATCH 127/188] HDFS-11534. Add counters for number of blocks in pending IBR. Contributed by Xiaobing Zhou. --- .../hdfs/server/datanode/BPServiceActor.java | 8 +- .../IncrementalBlockReportManager.java | 41 ++++- .../datanode/metrics/DataNodeMetrics.java | 35 +++++ .../TestBlockCountersInPendingIBR.java | 146 ++++++++++++++++++ 4 files changed, 223 insertions(+), 7 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockCountersInPendingIBR.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java index ee5eb72d7eb..ddc28b72a7a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java @@ -126,7 +126,9 @@ class BPServiceActor implements Runnable { this.initialRegistrationComplete = lifelineNnAddr != null ? new CountDownLatch(1) : null; this.dnConf = dn.getDnConf(); - this.ibrManager = new IncrementalBlockReportManager(dnConf.ibrInterval); + this.ibrManager = new IncrementalBlockReportManager( + dnConf.ibrInterval, + dn.getMetrics()); prevBlockReportId = ThreadLocalRandom.current().nextLong(); scheduler = new Scheduler(dnConf.heartBeatInterval, dnConf.getLifelineIntervalMs(), dnConf.blockReportInterval, @@ -350,7 +352,7 @@ class BPServiceActor implements Runnable { // or we will report an RBW replica after the BlockReport already reports // a FINALIZED one. ibrManager.sendIBRs(bpNamenode, bpRegistration, - bpos.getBlockPoolId(), dn.getMetrics()); + bpos.getBlockPoolId()); long brCreateStartTime = monotonicNow(); Map perVolumeBlockLists = @@ -678,7 +680,7 @@ class BPServiceActor implements Runnable { } if (ibrManager.sendImmediately() || sendHeartbeat) { ibrManager.sendIBRs(bpNamenode, bpRegistration, - bpos.getBlockPoolId(), dn.getMetrics()); + bpos.getBlockPoolId()); } List cmds = null; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/IncrementalBlockReportManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/IncrementalBlockReportManager.java index e95142db872..1779374f573 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/IncrementalBlockReportManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/IncrementalBlockReportManager.java @@ -52,6 +52,11 @@ class IncrementalBlockReportManager { /** The blocks in this IBR. */ final Map blocks = Maps.newHashMap(); + private DataNodeMetrics dnMetrics; + PerStorageIBR(final DataNodeMetrics dnMetrics) { + this.dnMetrics = dnMetrics; + } + /** * Remove the given block from this IBR * @return true if the block was removed; otherwise, return false. @@ -76,6 +81,25 @@ class IncrementalBlockReportManager { /** Put the block to this IBR. */ void put(ReceivedDeletedBlockInfo rdbi) { blocks.put(rdbi.getBlock(), rdbi); + increaseBlocksCounter(rdbi); + } + + private void increaseBlocksCounter( + final ReceivedDeletedBlockInfo receivedDeletedBlockInfo) { + switch (receivedDeletedBlockInfo.getStatus()) { + case RECEIVING_BLOCK: + dnMetrics.incrBlocksReceivingInPendingIBR(); + break; + case RECEIVED_BLOCK: + dnMetrics.incrBlocksReceivedInPendingIBR(); + break; + case DELETED_BLOCK: + dnMetrics.incrBlocksDeletedInPendingIBR(); + break; + default: + break; + } + dnMetrics.incrBlocksInPendingIBR(); } /** @@ -114,10 +138,14 @@ class IncrementalBlockReportManager { /** The timestamp of the last IBR. */ private volatile long lastIBR; + private DataNodeMetrics dnMetrics; - IncrementalBlockReportManager(final long ibrInterval) { + IncrementalBlockReportManager( + final long ibrInterval, + final DataNodeMetrics dnMetrics) { this.ibrInterval = ibrInterval; this.lastIBR = monotonicNow() - ibrInterval; + this.dnMetrics = dnMetrics; } boolean sendImmediately() { @@ -147,6 +175,10 @@ class IncrementalBlockReportManager { reports.add(new StorageReceivedDeletedBlocks(entry.getKey(), rdbi)); } } + + /* set blocks to zero */ + this.dnMetrics.resetBlocksInPendingIBR(); + readyToSend = false; return reports.toArray(new StorageReceivedDeletedBlocks[reports.size()]); } @@ -162,7 +194,7 @@ class IncrementalBlockReportManager { /** Send IBRs to namenode. */ void sendIBRs(DatanodeProtocol namenode, DatanodeRegistration registration, - String bpid, DataNodeMetrics metrics) throws IOException { + String bpid) throws IOException { // Generate a list of the pending reports for each storage under the lock final StorageReceivedDeletedBlocks[] reports = generateIBRs(); if (reports.length == 0) { @@ -180,8 +212,9 @@ class IncrementalBlockReportManager { namenode.blockReceivedAndDeleted(registration, bpid, reports); success = true; } finally { - metrics.addIncrementalBlockReport(monotonicNow() - startTime); + if (success) { + dnMetrics.addIncrementalBlockReport(monotonicNow() - startTime); lastIBR = startTime; } else { // If we didn't succeed in sending the report, put all of the @@ -199,7 +232,7 @@ class IncrementalBlockReportManager { // This is the first time we are adding incremental BR state for // this storage so create a new map. This is required once per // storage, per service actor. - perStorage = new PerStorageIBR(); + perStorage = new PerStorageIBR(dnMetrics); pendingIBRs.put(storage, perStorage); } return perStorage; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java index 0d82fed770d..a8a691980bb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java @@ -31,6 +31,7 @@ import org.apache.hadoop.metrics2.lib.MutableCounterLong; import org.apache.hadoop.metrics2.lib.MutableQuantiles; import org.apache.hadoop.metrics2.lib.MutableRate; import org.apache.hadoop.metrics2.lib.MutableGaugeInt; +import org.apache.hadoop.metrics2.lib.MutableGaugeLong; import org.apache.hadoop.metrics2.source.JvmMetrics; import java.util.concurrent.ThreadLocalRandom; @@ -130,6 +131,14 @@ public class DataNodeMetrics { @Metric MutableRate sendDataPacketTransferNanos; final MutableQuantiles[] sendDataPacketTransferNanosQuantiles; + @Metric("Count of blocks in pending IBR") + private MutableGaugeLong blocksInPendingIBR; + @Metric("Count of blocks at receiving status in pending IBR") + private MutableGaugeLong blocksReceivingInPendingIBR; + @Metric("Count of blocks at received status in pending IBR") + private MutableGaugeLong blocksReceivedInPendingIBR; + @Metric("Count of blocks at deleted status in pending IBR") + private MutableGaugeLong blocksDeletedInPendingIBR; @Metric("Count of erasure coding reconstruction tasks") MutableCounterLong ecReconstructionTasks; @Metric("Count of erasure coding failed reconstruction tasks") @@ -433,6 +442,32 @@ public class DataNodeMetrics { } } + /** + * Resets blocks in pending IBR to zero. + */ + public void resetBlocksInPendingIBR() { + blocksInPendingIBR.set(0); + blocksReceivingInPendingIBR.set(0); + blocksReceivedInPendingIBR.set(0); + blocksDeletedInPendingIBR.set(0); + } + + public void incrBlocksInPendingIBR() { + blocksInPendingIBR.incr(); + } + + public void incrBlocksReceivingInPendingIBR() { + blocksReceivingInPendingIBR.incr(); + } + + public void incrBlocksReceivedInPendingIBR() { + blocksReceivedInPendingIBR.incr(); + } + + public void incrBlocksDeletedInPendingIBR() { + blocksDeletedInPendingIBR.incr(); + } + public void incrECReconstructionTasks() { ecReconstructionTasks.incr(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockCountersInPendingIBR.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockCountersInPendingIBR.java new file mode 100644 index 00000000000..43d39a6924c --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockCountersInPendingIBR.java @@ -0,0 +1,146 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.datanode; + +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.timeout; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.client.BlockReportOptions; +import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB; +import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; +import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; +import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo; +import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks; +import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo.BlockStatus; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; +import org.apache.hadoop.test.MetricsAsserts; +import org.junit.Test; +import org.mockito.Mockito; + +/** + * Test counters for number of blocks in pending IBR. + */ +public class TestBlockCountersInPendingIBR { + + @Test + public void testBlockCounters() throws Exception { + final Configuration conf = new HdfsConfiguration(); + + /* + * Set a really long value for dfs.blockreport.intervalMsec and + * dfs.heartbeat.interval, so that incremental block reports and heartbeats + * won't be sent during this test unless they're triggered manually. + */ + conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10800000L); + conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1080L); + + final MiniDFSCluster cluster = + new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + cluster.waitActive(); + final DatanodeProtocolClientSideTranslatorPB spy = + InternalDataNodeTestUtils.spyOnBposToNN( + cluster.getDataNodes().get(0), cluster.getNameNode()); + final DataNode datanode = cluster.getDataNodes().get(0); + + /* We should get 0 incremental block report. */ + Mockito.verify(spy, timeout(60000).times(0)).blockReceivedAndDeleted( + any(DatanodeRegistration.class), + anyString(), + any(StorageReceivedDeletedBlocks[].class)); + + /* + * Create fake blocks notification on the DataNode. This will be sent with + * the next incremental block report. + */ + final BPServiceActor actor = + datanode.getAllBpOs().get(0).getBPServiceActors().get(0); + final FsDatasetSpi dataset = datanode.getFSDataset(); + final DatanodeStorage storage; + try (FsDatasetSpi.FsVolumeReferences volumes = + dataset.getFsVolumeReferences()) { + storage = dataset.getStorage(volumes.get(0).getStorageID()); + } + + ReceivedDeletedBlockInfo rdbi = null; + /* block at status of RECEIVING_BLOCK */ + rdbi = new ReceivedDeletedBlockInfo( + new Block(5678, 512, 1000), BlockStatus.RECEIVING_BLOCK, null); + actor.getIbrManager().addRDBI(rdbi, storage); + + /* block at status of RECEIVED_BLOCK */ + rdbi = new ReceivedDeletedBlockInfo( + new Block(5679, 512, 1000), BlockStatus.RECEIVED_BLOCK, null); + actor.getIbrManager().addRDBI(rdbi, storage); + + /* block at status of DELETED_BLOCK */ + rdbi = new ReceivedDeletedBlockInfo( + new Block(5680, 512, 1000), BlockStatus.DELETED_BLOCK, null); + actor.getIbrManager().addRDBI(rdbi, storage); + + /* verify counters before sending IBR */ + verifyBlockCounters(datanode, 3, 1, 1, 1); + + /* Manually trigger a block report. */ + datanode.triggerBlockReport( + new BlockReportOptions.Factory(). + setIncremental(true). + build() + ); + + /* + * triggerBlockReport returns before the block report is actually sent. Wait + * for it to be sent here. + */ + Mockito.verify(spy, timeout(60000).times(1)). + blockReceivedAndDeleted( + any(DatanodeRegistration.class), + anyString(), + any(StorageReceivedDeletedBlocks[].class)); + + /* verify counters after sending IBR */ + verifyBlockCounters(datanode, 0, 0, 0, 0); + + cluster.shutdown(); + } + + + private void verifyBlockCounters(final DataNode datanode, + final long blocksInPendingIBR, final long blocksReceivingInPendingIBR, + final long blocksReceivedInPendingIBR, + final long blocksDeletedInPendingIBR) { + + final MetricsRecordBuilder m = MetricsAsserts + .getMetrics(datanode.getMetrics().name()); + + MetricsAsserts.assertGauge("BlocksInPendingIBR", + blocksInPendingIBR, m); + MetricsAsserts.assertGauge("BlocksReceivingInPendingIBR", + blocksReceivingInPendingIBR, m); + MetricsAsserts.assertGauge("BlocksReceivedInPendingIBR", + blocksReceivedInPendingIBR, m); + MetricsAsserts.assertGauge("BlocksDeletedInPendingIBR", + blocksDeletedInPendingIBR, m); + } +} From 1f66524432edf37e28ec03aba8d12b22ce0ef60c Mon Sep 17 00:00:00 2001 From: Hanisha Koneru Date: Fri, 24 Mar 2017 14:44:25 -0700 Subject: [PATCH 128/188] HDFS-11570. Unit test for NameNodeStatusMXBean. Contributed by Hanisha Koneru. --- .../namenode/TestNameNodeStatusMXBean.java | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeStatusMXBean.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeStatusMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeStatusMXBean.java new file mode 100644 index 00000000000..c03dc209f9d --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeStatusMXBean.java @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.server.datanode.TestDataNodeMXBean; +import org.junit.Assert; +import org.junit.Test; + +import javax.management.MBeanServer; +import javax.management.ObjectName; +import java.lang.management.ManagementFactory; + +/** + * Class for testing {@link NameNodeStatusMXBean} implementation. + */ +public class TestNameNodeStatusMXBean { + + public static final Log LOG = LogFactory.getLog( + TestNameNodeStatusMXBean.class); + + @Test(timeout = 120000L) + public void testDataNodeMXBean() throws Exception { + Configuration conf = new Configuration(); + MiniDFSCluster cluster = null; + + try { + cluster = new MiniDFSCluster.Builder(conf).build(); + cluster.waitActive(); + + NameNode nn = cluster.getNameNode(); + + MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); + ObjectName mxbeanName = new ObjectName( + "Hadoop:service=NameNode,name=NameNodeStatus"); + + // Get attribute "NNRole" + String nnRole = (String)mbs.getAttribute(mxbeanName, "NNRole"); + Assert.assertEquals(nn.getNNRole(), nnRole); + + // Get attribute "State" + String state = (String)mbs.getAttribute(mxbeanName, "State"); + Assert.assertEquals(nn.getState(), state); + + // Get attribute "HostAndPort" + String hostAndPort = (String)mbs.getAttribute(mxbeanName, "HostAndPort"); + Assert.assertEquals(nn.getHostAndPort(), hostAndPort); + + // Get attribute "SecurityEnabled" + boolean securityEnabled = (boolean)mbs.getAttribute(mxbeanName, + "SecurityEnabled"); + Assert.assertEquals(nn.isSecurityEnabled(), securityEnabled); + + // Get attribute "LastHATransitionTime" + long lastHATransitionTime = (long)mbs.getAttribute(mxbeanName, + "LastHATransitionTime"); + Assert.assertEquals(nn.getLastHATransitionTime(), lastHATransitionTime); + + // Get attribute "BytesWithFutureGenerationStamps" + long bytesWithFutureGenerationStamps = (long)mbs.getAttribute( + mxbeanName, "BytesWithFutureGenerationStamps"); + Assert.assertEquals(nn.getBytesWithFutureGenerationStamps(), + bytesWithFutureGenerationStamps); + + // Get attribute "SlowPeersReport" + String slowPeersReport = (String)mbs.getAttribute(mxbeanName, + "SlowPeersReport"); + Assert.assertEquals(nn.getSlowPeersReport(), slowPeersReport); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } +} From 84ddedc0b2d58257d45c16ee5e83b15f94a7ba3a Mon Sep 17 00:00:00 2001 From: Tsuyoshi Ozawa Date: Sat, 25 Mar 2017 10:06:52 +0900 Subject: [PATCH 129/188] HADOOP-10101. Update guava dependency to the latest version. (ozawa) --- .../main/java/org/apache/hadoop/fs/shell/XAttrCommands.java | 2 +- .../java/org/apache/hadoop/ha/ZKFailoverController.java | 3 +-- .../java/org/apache/hadoop/metrics2/AbstractMetric.java | 3 ++- .../main/java/org/apache/hadoop/metrics2/MetricsTag.java | 3 ++- .../apache/hadoop/metrics2/impl/AbstractMetricsRecord.java | 3 ++- .../main/java/org/apache/hadoop/metrics2/impl/MsInfo.java | 4 ++-- .../org/apache/hadoop/metrics2/lib/MetricsInfoImpl.java | 3 ++- .../org/apache/hadoop/metrics2/lib/MetricsRegistry.java | 4 ++-- .../org/apache/hadoop/metrics2/source/JvmMetricsInfo.java | 6 +++--- .../java/org/apache/hadoop/metrics2/util/MetricsCache.java | 4 ++-- .../java/org/apache/hadoop/util/curator/ChildReaper.java | 4 ++-- .../org/apache/hadoop/hdfs/DFSInotifyEventInputStream.java | 4 ++-- .../org/apache/hadoop/hdfs/server/datanode/DataStorage.java | 2 +- .../hadoop/hdfs/server/namenode/AclTransformation.java | 4 ++-- .../org/apache/hadoop/hdfs/server/namenode/JournalSet.java | 2 +- .../hadoop/hdfs/qjournal/client/TestQJMWithFaults.java | 2 +- .../hdfs/qjournal/client/TestQuorumJournalManager.java | 2 +- hadoop-project/pom.xml | 2 +- .../records/impl/pb/ApplicationSubmissionContextPBImpl.java | 2 +- .../src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java | 4 ++-- .../hadoop/yarn/server/resourcemanager/RMAppManager.java | 6 +++--- .../scheduler/fair/AllocationFileLoaderService.java | 2 +- .../server/resourcemanager/scheduler/fair/QueueManager.java | 2 +- 23 files changed, 38 insertions(+), 35 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/XAttrCommands.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/XAttrCommands.java index d55c80b1a7e..4505aa90cd0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/XAttrCommands.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/XAttrCommands.java @@ -67,7 +67,7 @@ class XAttrCommands extends FsCommand { "0x and 0s, respectively.\n" + ": The file or directory.\n"; private final static Function enValueOfFunc = - Enums.valueOfFunction(XAttrCodec.class); + Enums.stringConverter(XAttrCodec.class); private String name = null; private boolean dump = false; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java index 0ed91582c9a..055bcaa5825 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java @@ -55,7 +55,6 @@ import org.apache.zookeeper.data.ACL; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; -import com.google.common.base.Throwables; import com.google.common.util.concurrent.ThreadFactoryBuilder; @InterfaceAudience.LimitedPrivate("HDFS") @@ -511,7 +510,7 @@ public abstract class ZKFailoverController { doFence(target); } catch (Throwable t) { recordActiveAttempt(new ActiveAttemptRecord(false, "Unable to fence old active: " + StringUtils.stringifyException(t))); - Throwables.propagate(t); + throw t; } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/AbstractMetric.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/AbstractMetric.java index 6a11b875cd4..0605156a86a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/AbstractMetric.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/AbstractMetric.java @@ -18,6 +18,7 @@ package org.apache.hadoop.metrics2; +import com.google.common.base.MoreObjects; import com.google.common.base.Objects; import static com.google.common.base.Preconditions.*; @@ -84,7 +85,7 @@ public abstract class AbstractMetric implements MetricsInfo { } @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("info", info) .add("value", value()) .toString(); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsTag.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsTag.java index e9e50a4acba..68b07379c50 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsTag.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsTag.java @@ -18,6 +18,7 @@ package org.apache.hadoop.metrics2; +import com.google.common.base.MoreObjects; import com.google.common.base.Objects; import static com.google.common.base.Preconditions.*; @@ -80,7 +81,7 @@ public class MetricsTag implements MetricsInfo { } @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("info", info) .add("value", value()) .toString(); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/AbstractMetricsRecord.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/AbstractMetricsRecord.java index 3684c7ef5c3..fec29c2b4ce 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/AbstractMetricsRecord.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/AbstractMetricsRecord.java @@ -18,6 +18,7 @@ package org.apache.hadoop.metrics2.impl; +import com.google.common.base.MoreObjects; import com.google.common.base.Objects; import com.google.common.collect.Iterables; @@ -43,7 +44,7 @@ abstract class AbstractMetricsRecord implements MetricsRecord { } @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("timestamp", timestamp()) .add("name", name()) .add("description", description()) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MsInfo.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MsInfo.java index 782f7557207..5de7edcb0dd 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MsInfo.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MsInfo.java @@ -18,7 +18,7 @@ package org.apache.hadoop.metrics2.impl; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.metrics2.MetricsInfo; @@ -48,7 +48,7 @@ public enum MsInfo implements MetricsInfo { } @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("name", name()).add("description", desc) .toString(); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsInfoImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsInfoImpl.java index dfb6c357a22..054f21147ce 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsInfoImpl.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsInfoImpl.java @@ -18,6 +18,7 @@ package org.apache.hadoop.metrics2.lib; +import com.google.common.base.MoreObjects; import com.google.common.base.Objects; import static com.google.common.base.Preconditions.*; import org.apache.hadoop.metrics2.MetricsInfo; @@ -55,7 +56,7 @@ class MetricsInfoImpl implements MetricsInfo { } @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("name", name).add("description", description) .toString(); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsRegistry.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsRegistry.java index 1ef74f41c03..0af45a69427 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsRegistry.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsRegistry.java @@ -22,7 +22,7 @@ import java.util.Collection; import java.util.Map; import com.google.common.collect.Maps; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -416,7 +416,7 @@ public class MetricsRegistry { } @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("info", metricsInfo).add("tags", tags()).add("metrics", metrics()) .toString(); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetricsInfo.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetricsInfo.java index 55bb41720ed..59a79fd4d67 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetricsInfo.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetricsInfo.java @@ -18,11 +18,11 @@ package org.apache.hadoop.metrics2.source; -import com.google.common.base.Objects; - import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.metrics2.MetricsInfo; +import com.google.common.base.MoreObjects; + /** * JVM and logging related metrics info instances */ @@ -60,7 +60,7 @@ public enum JvmMetricsInfo implements MetricsInfo { @Override public String description() { return desc; } @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("name", name()).add("description", desc) .toString(); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/MetricsCache.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/MetricsCache.java index efcb286fae2..753e307cefa 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/MetricsCache.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/MetricsCache.java @@ -31,7 +31,7 @@ import org.apache.hadoop.metrics2.AbstractMetric; import org.apache.hadoop.metrics2.MetricsRecord; import org.apache.hadoop.metrics2.MetricsTag; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.common.collect.Maps; /** @@ -127,7 +127,7 @@ public class MetricsCache { } @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("tags", tags).add("metrics", metrics) .toString(); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ChildReaper.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ChildReaper.java index 3bff187a287..86142fb6d3a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ChildReaper.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ChildReaper.java @@ -19,7 +19,6 @@ package org.apache.hadoop.util.curator; import com.google.common.base.Preconditions; -import com.google.common.collect.Sets; import org.apache.curator.framework.recipes.locks.Reaper; import org.apache.curator.utils.CloseableUtils; import org.apache.curator.framework.CuratorFramework; @@ -34,6 +33,7 @@ import org.slf4j.LoggerFactory; import java.io.Closeable; import java.io.IOException; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -82,7 +82,7 @@ public class ChildReaper implements Closeable * @since 15.0 */ public static Set newConcurrentHashSet() { - return Sets.newSetFromMap(new ConcurrentHashMap()); + return Collections.newSetFromMap(new ConcurrentHashMap()); } private enum State diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInotifyEventInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInotifyEventInputStream.java index 45bea5ef5f5..bcdc1c8285d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInotifyEventInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInotifyEventInputStream.java @@ -18,7 +18,7 @@ package org.apache.hadoop.hdfs; -import com.google.common.collect.Iterators; +import java.util.Collections; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.hdfs.inotify.EventBatch; @@ -72,7 +72,7 @@ public class DFSInotifyEventInputStream { DFSInotifyEventInputStream(ClientProtocol namenode, Tracer tracer, long lastReadTxid) { this.namenode = namenode; - this.it = Iterators.emptyIterator(); + this.it = Collections.emptyIterator(); this.lastReadTxid = lastReadTxid; this.tracer = tracer; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java index a5324be841f..57ef1bcec99 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java @@ -1109,7 +1109,7 @@ public class DataStorage extends Storage { } linkWorkers.shutdown(); for (Future f : futures) { - Futures.get(f, IOException.class); + Futures.getChecked(f, IOException.class); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclTransformation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclTransformation.java index c887e9dcb58..3e4a319d56e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclTransformation.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclTransformation.java @@ -28,7 +28,7 @@ import java.util.EnumSet; import java.util.Iterator; import java.util.List; -import com.google.common.base.Objects; +import com.google.common.base.MoreObjects; import com.google.common.collect.ComparisonChain; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -366,7 +366,7 @@ final class AclTransformation { for (AclEntry entry: aclBuilder) { scopeFound.add(entry.getScope()); if (entry.getType() == GROUP || entry.getName() != null) { - FsAction scopeUnionPerms = Objects.firstNonNull( + FsAction scopeUnionPerms = MoreObjects.firstNonNull( unionPerms.get(entry.getScope()), FsAction.NONE); unionPerms.put(entry.getScope(), scopeUnionPerms.or(entry.getPermission())); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java index fde54a8a43f..db77d31be20 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalSet.java @@ -63,7 +63,7 @@ public class JournalSet implements JournalManager { public int compare(EditLogInputStream elis1, EditLogInputStream elis2) { // we want local logs to be ordered earlier in the collection, and true // is considered larger than false, so we want to invert the booleans here - return ComparisonChain.start().compare(!elis1.isLocalLog(), + return ComparisonChain.start().compareFalseFirst(!elis1.isLocalLog(), !elis2.isLocalLog()).result(); } }; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java index ecdbaf5296d..c752f239e56 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java @@ -402,7 +402,7 @@ public class TestQJMWithFaults { @Override protected ExecutorService createSingleThreadExecutor() { - return MoreExecutors.sameThreadExecutor(); + return MoreExecutors.newDirectExecutorService(); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java index 7d770e067ab..9aada1d1550 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java @@ -946,7 +946,7 @@ public class TestQuorumJournalManager { protected ExecutorService createSingleThreadExecutor() { // Don't parallelize calls to the quorum in the tests. // This makes the tests more deterministic. - return MoreExecutors.sameThreadExecutor(); + return MoreExecutors.newDirectExecutorService(); } }; diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 8de7703359a..68e8b5c296b 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -514,7 +514,7 @@ com.google.guava guava - 11.0.2 + 21.0 com.google.code.gson diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java index 62b54e7ed8e..66930e74bf8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java @@ -283,7 +283,7 @@ extends ApplicationSubmissionContext { "maximum allowed length of a tag is " + YarnConfiguration.APPLICATION_MAX_TAG_LENGTH); } - if (!CharMatcher.ASCII.matchesAllOf(tag)) { + if (!CharMatcher.ascii().matchesAllOf(tag)) { throw new IllegalArgumentException("A tag can only have ASCII " + "characters! Invalid tag - " + tag); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java index de6a52bda52..300bf3ee457 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java @@ -275,7 +275,7 @@ public abstract class WebApp extends ServletModule { static String getPrefix(String pathSpec) { int start = 0; - while (CharMatcher.WHITESPACE.matches(pathSpec.charAt(start))) { + while (CharMatcher.whitespace().matches(pathSpec.charAt(start))) { ++start; } if (pathSpec.charAt(start) != '/') { @@ -291,7 +291,7 @@ public abstract class WebApp extends ServletModule { char c; do { c = pathSpec.charAt(--ci); - } while (c == '/' || CharMatcher.WHITESPACE.matches(c)); + } while (c == '/' || CharMatcher.whitespace().matches(c)); return pathSpec.substring(start, ci + 1); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index e211867f852..1f446e6d48b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -590,7 +590,7 @@ public class RMAppManager implements EventHandler, this.rmContext.getStateStore() .updateApplicationStateSynchronously(appState, false, future); - Futures.get(future, YarnException.class); + Futures.getChecked(future, YarnException.class); // update in-memory ((RMAppImpl) app).updateApplicationTimeout(newExpireTime); @@ -627,7 +627,7 @@ public class RMAppManager implements EventHandler, return; } - Futures.get(future, YarnException.class); + Futures.getChecked(future, YarnException.class); // update in-memory ((RMAppImpl) app).setApplicationPriority(appPriority); @@ -710,7 +710,7 @@ public class RMAppManager implements EventHandler, false, future); try { - Futures.get(future, YarnException.class); + Futures.getChecked(future, YarnException.class); } catch (YarnException ex) { if (!toSuppressException) { throw ex; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java index 163a265918e..d29d34e0944 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java @@ -465,7 +465,7 @@ public class AllocationFileLoaderService extends AbstractService { Set reservableQueues, Set nonPreemptableQueues) throws AllocationConfigurationException { - String queueName = CharMatcher.WHITESPACE.trimFrom( + String queueName = CharMatcher.whitespace().trimFrom( element.getAttribute("name")); if (queueName.contains(".")) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java index 3c601fa1a11..5b006dfcf99 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java @@ -535,6 +535,6 @@ public class QueueManager { // use the same white space trim as in QueueMetrics() otherwise things fail // guava uses a different definition for whitespace than java. return !node.isEmpty() && - node.equals(CharMatcher.WHITESPACE.trimFrom(node)); + node.equals(CharMatcher.whitespace().trimFrom(node)); } } From cffea251be4b73ca16e5e11f0be2d22651330f73 Mon Sep 17 00:00:00 2001 From: Sunil G Date: Sat, 25 Mar 2017 22:26:19 +0530 Subject: [PATCH 130/188] YARN-5956. Refactor ClientRMService for unify error handling across apis. Contributed by Kai Sasaki. --- .../resourcemanager/ClientRMService.java | 286 ++++++------------ .../server/resourcemanager/RMAuditLogger.java | 5 + 2 files changed, 104 insertions(+), 187 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 929a9e7b672..6130b33736d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -196,7 +196,7 @@ public class ClientRMService extends AbstractService implements protected RMDelegationTokenSecretManager rmDTSecretManager; private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); - InetSocketAddress clientBindAddress; + private InetSocketAddress clientBindAddress; private final ApplicationACLsManager applicationsACLsManager; private final QueueACLsManager queueACLsManager; @@ -206,9 +206,6 @@ public class ClientRMService extends AbstractService implements private ReservationSystem reservationSystem; private ReservationInputValidator rValidator; - private static final EnumSet COMPLETED_APP_STATES = EnumSet.of( - RMAppState.FINISHED, RMAppState.FINISHING, RMAppState.FAILED, - RMAppState.KILLED, RMAppState.FINAL_SAVING, RMAppState.KILLING); private static final EnumSet ACTIVE_APP_STATES = EnumSet.of( RMAppState.ACCEPTED, RMAppState.RUNNING); @@ -298,11 +295,12 @@ public class ClientRMService extends AbstractService implements /** * check if the calling user has the access to application information. - * @param callerUGI - * @param owner - * @param operationPerformed - * @param application - * @return + * @param callerUGI the user information who submit the request + * @param owner the user of the application + * @param operationPerformed the type of operation defined in + * {@link ApplicationAccessType} + * @param application submitted application + * @return access is permitted or not */ private boolean checkAccess(UserGroupInformation callerUGI, String owner, ApplicationAccessType operationPerformed, RMApp application) { @@ -379,24 +377,14 @@ public class ClientRMService extends AbstractService implements public GetApplicationAttemptReportResponse getApplicationAttemptReport( GetApplicationAttemptReportRequest request) throws YarnException, IOException { + ApplicationId applicationId + = request.getApplicationAttemptId().getApplicationId(); ApplicationAttemptId appAttemptId = request.getApplicationAttemptId(); - UserGroupInformation callerUGI; - try { - callerUGI = UserGroupInformation.getCurrentUser(); - } catch (IOException ie) { - LOG.info("Error getting UGI ", ie); - throw RPCUtil.getRemoteException(ie); - } - RMApp application = this.rmContext.getRMApps().get( - appAttemptId.getApplicationId()); - if (application == null) { - // If the RM doesn't have the application, throw - // ApplicationNotFoundException and let client to handle. - throw new ApplicationNotFoundException("Application with id '" - + request.getApplicationAttemptId().getApplicationId() - + "' doesn't exist in RM. Please check that the job " - + "submission was successful."); - } + UserGroupInformation callerUGI = getCallerUgi(applicationId, + AuditConstants.GET_APP_ATTEMPT_REPORT); + RMApp application = verifyUserAccessForRMApp(applicationId, callerUGI, + AuditConstants.GET_APP_ATTEMPT_REPORT, ApplicationAccessType.VIEW_APP, + false); boolean allowAccess = checkAccess(callerUGI, application.getUser(), ApplicationAccessType.VIEW_APP, application); @@ -422,21 +410,11 @@ public class ClientRMService extends AbstractService implements public GetApplicationAttemptsResponse getApplicationAttempts( GetApplicationAttemptsRequest request) throws YarnException, IOException { ApplicationId appId = request.getApplicationId(); - UserGroupInformation callerUGI; - try { - callerUGI = UserGroupInformation.getCurrentUser(); - } catch (IOException ie) { - LOG.info("Error getting UGI ", ie); - throw RPCUtil.getRemoteException(ie); - } - RMApp application = this.rmContext.getRMApps().get(appId); - if (application == null) { - // If the RM doesn't have the application, throw - // ApplicationNotFoundException and let client to handle. - throw new ApplicationNotFoundException("Application with id '" + appId - + "' doesn't exist in RM. Please check that the job submission " - + "was successful."); - } + UserGroupInformation callerUGI = getCallerUgi(appId, + AuditConstants.GET_APP_ATTEMPTS); + RMApp application = verifyUserAccessForRMApp(appId, callerUGI, + AuditConstants.GET_APP_ATTEMPTS, ApplicationAccessType.VIEW_APP, + false); boolean allowAccess = checkAccess(callerUGI, application.getUser(), ApplicationAccessType.VIEW_APP, application); GetApplicationAttemptsResponse response = null; @@ -471,21 +449,11 @@ public class ClientRMService extends AbstractService implements ContainerId containerId = request.getContainerId(); ApplicationAttemptId appAttemptId = containerId.getApplicationAttemptId(); ApplicationId appId = appAttemptId.getApplicationId(); - UserGroupInformation callerUGI; - try { - callerUGI = UserGroupInformation.getCurrentUser(); - } catch (IOException ie) { - LOG.info("Error getting UGI ", ie); - throw RPCUtil.getRemoteException(ie); - } - RMApp application = this.rmContext.getRMApps().get(appId); - if (application == null) { - // If the RM doesn't have the application, throw - // ApplicationNotFoundException and let client to handle. - throw new ApplicationNotFoundException("Application with id '" + appId - + "' doesn't exist in RM. Please check that the job submission " - + "was successful."); - } + UserGroupInformation callerUGI = getCallerUgi(appId, + AuditConstants.GET_CONTAINER_REPORT); + RMApp application = verifyUserAccessForRMApp(appId, callerUGI, + AuditConstants.GET_CONTAINER_REPORT, ApplicationAccessType.VIEW_APP, + false); boolean allowAccess = checkAccess(callerUGI, application.getUser(), ApplicationAccessType.VIEW_APP, application); GetContainerReportResponse response = null; @@ -496,13 +464,13 @@ public class ClientRMService extends AbstractService implements "ApplicationAttempt with id '" + appAttemptId + "' doesn't exist in RM."); } - RMContainer rmConatiner = this.rmContext.getScheduler().getRMContainer( + RMContainer rmContainer = this.rmContext.getScheduler().getRMContainer( containerId); - if (rmConatiner == null) { + if (rmContainer == null) { throw new ContainerNotFoundException("Container with id '" + containerId + "' doesn't exist in RM."); } - response = GetContainerReportResponse.newInstance(rmConatiner + response = GetContainerReportResponse.newInstance(rmContainer .createContainerReport()); } else { throw new YarnException("User " + callerUGI.getShortUserName() @@ -522,21 +490,10 @@ public class ClientRMService extends AbstractService implements throws YarnException, IOException { ApplicationAttemptId appAttemptId = request.getApplicationAttemptId(); ApplicationId appId = appAttemptId.getApplicationId(); - UserGroupInformation callerUGI; - try { - callerUGI = UserGroupInformation.getCurrentUser(); - } catch (IOException ie) { - LOG.info("Error getting UGI ", ie); - throw RPCUtil.getRemoteException(ie); - } - RMApp application = this.rmContext.getRMApps().get(appId); - if (application == null) { - // If the RM doesn't have the application, throw - // ApplicationNotFoundException and let client to handle. - throw new ApplicationNotFoundException("Application with id '" + appId - + "' doesn't exist in RM. Please check that the job submission " - + "was successful."); - } + UserGroupInformation callerUGI = getCallerUgi(appId, + AuditConstants.GET_CONTAINERS); + RMApp application = verifyUserAccessForRMApp(appId, callerUGI, + AuditConstants.GET_CONTAINERS, ApplicationAccessType.VIEW_APP, false); boolean allowAccess = checkAccess(callerUGI, application.getUser(), ApplicationAccessType.VIEW_APP, application); GetContainersResponse response = null; @@ -600,6 +557,7 @@ public class ClientRMService extends AbstractService implements TimelineUtils.FLOW_RUN_ID_TAG_PREFIX.toLowerCase() + ":")) { value = tag.substring(TimelineUtils.FLOW_RUN_ID_TAG_PREFIX.length() + 1); + // In order to check the number format Long.valueOf(value); } } @@ -676,9 +634,8 @@ public class ClientRMService extends AbstractService implements throw e; } - SubmitApplicationResponse response = recordFactory + return recordFactory .newRecordInstance(SubmitApplicationResponse.class); - return response; } @SuppressWarnings("unchecked") @@ -689,26 +646,11 @@ public class ClientRMService extends AbstractService implements ApplicationAttemptId attemptId = request.getApplicationAttemptId(); ApplicationId applicationId = attemptId.getApplicationId(); - UserGroupInformation callerUGI; - try { - callerUGI = UserGroupInformation.getCurrentUser(); - } catch (IOException ie) { - LOG.info("Error getting UGI ", ie); - RMAuditLogger.logFailure("UNKNOWN", AuditConstants.FAIL_ATTEMPT_REQUEST, - "UNKNOWN", "ClientRMService" , "Error getting UGI", - applicationId, attemptId); - throw RPCUtil.getRemoteException(ie); - } - - RMApp application = this.rmContext.getRMApps().get(applicationId); - if (application == null) { - RMAuditLogger.logFailure(callerUGI.getUserName(), - AuditConstants.FAIL_ATTEMPT_REQUEST, "UNKNOWN", "ClientRMService", - "Trying to fail an attempt of an absent application", applicationId, - attemptId); - throw new ApplicationNotFoundException("Trying to fail an attempt " - + attemptId + " of an absent application " + applicationId); - } + UserGroupInformation callerUGI = getCallerUgi(applicationId, + AuditConstants.FAIL_ATTEMPT_REQUEST); + RMApp application = verifyUserAccessForRMApp(applicationId, callerUGI, + AuditConstants.FAIL_ATTEMPT_REQUEST, ApplicationAccessType.MODIFY_APP, + true); RMAppAttempt appAttempt = application.getAppAttempts().get(attemptId); if (appAttempt == null) { @@ -716,28 +658,14 @@ public class ClientRMService extends AbstractService implements "ApplicationAttempt with id '" + attemptId + "' doesn't exist in RM."); } - if (!checkAccess(callerUGI, application.getUser(), - ApplicationAccessType.MODIFY_APP, application)) { - RMAuditLogger.logFailure(callerUGI.getShortUserName(), - AuditConstants.FAIL_ATTEMPT_REQUEST, - "User doesn't have permissions to " - + ApplicationAccessType.MODIFY_APP.toString(), "ClientRMService", - AuditConstants.UNAUTHORIZED_USER, applicationId); - throw RPCUtil.getRemoteException(new AccessControlException("User " - + callerUGI.getShortUserName() + " cannot perform operation " - + ApplicationAccessType.MODIFY_APP.name() + " on " + applicationId)); - } - FailApplicationAttemptResponse response = recordFactory.newRecordInstance(FailApplicationAttemptResponse.class); - if (!ACTIVE_APP_STATES.contains(application.getState())) { - if (COMPLETED_APP_STATES.contains(application.getState())) { - RMAuditLogger.logSuccess(callerUGI.getShortUserName(), - AuditConstants.FAIL_ATTEMPT_REQUEST, "ClientRMService", - applicationId); - return response; - } + if (application.isAppInCompletedStates()) { + RMAuditLogger.logSuccess(callerUGI.getShortUserName(), + AuditConstants.FAIL_ATTEMPT_REQUEST, "ClientRMService", + applicationId); + return response; } this.rmContext.getDispatcher().getEventHandler().handle( @@ -765,11 +693,10 @@ public class ClientRMService extends AbstractService implements } catch (IOException ie) { LOG.info("Error getting UGI ", ie); RMAuditLogger.logFailure("UNKNOWN", AuditConstants.KILL_APP_REQUEST, - "UNKNOWN", "ClientRMService" , "Error getting UGI", - applicationId, callerContext); + "UNKNOWN", "ClientRMService", "Error getting UGI", + applicationId, callerContext); throw RPCUtil.getRemoteException(ie); } - RMApp application = this.rmContext.getRMApps().get(applicationId); if (application == null) { RMAuditLogger.logFailure(callerUGI.getUserName(), @@ -815,7 +742,7 @@ public class ClientRMService extends AbstractService implements .handle(new RMAppKillByClientEvent(applicationId, message.toString(), callerUGI, remoteAddress)); - // For UnmanagedAMs, return true so they don't retry + // For Unmanaged AMs, return true so they don't retry return KillApplicationResponse.newInstance( application.getApplicationSubmissionContext().getUnmanagedAM()); } @@ -1107,15 +1034,15 @@ public class ClientRMService extends AbstractService implements RMDelegationTokenIdentifier tokenIdentifier = new RMDelegationTokenIdentifier(owner, new Text(request.getRenewer()), realUser); - Token realRMDTtoken = + Token realRMDToken = new Token(tokenIdentifier, this.rmDTSecretManager); response.setRMDelegationToken( BuilderUtils.newDelegationToken( - realRMDTtoken.getIdentifier(), - realRMDTtoken.getKind().toString(), - realRMDTtoken.getPassword(), - realRMDTtoken.getService().toString() + realRMDToken.getIdentifier(), + realRMDToken.getKind().toString(), + realRMDToken.getPassword(), + realRMDToken.getService().toString() )); return response; } catch(IOException io) { @@ -1175,37 +1102,11 @@ public class ClientRMService extends AbstractService implements MoveApplicationAcrossQueuesRequest request) throws YarnException { ApplicationId applicationId = request.getApplicationId(); - UserGroupInformation callerUGI; - try { - callerUGI = UserGroupInformation.getCurrentUser(); - } catch (IOException ie) { - LOG.info("Error getting UGI ", ie); - RMAuditLogger.logFailure("UNKNOWN", AuditConstants.MOVE_APP_REQUEST, - "UNKNOWN", "ClientRMService" , "Error getting UGI", - applicationId); - throw RPCUtil.getRemoteException(ie); - } - - RMApp application = this.rmContext.getRMApps().get(applicationId); - if (application == null) { - RMAuditLogger.logFailure(callerUGI.getUserName(), - AuditConstants.MOVE_APP_REQUEST, "UNKNOWN", "ClientRMService", - "Trying to move an absent application", applicationId); - throw new ApplicationNotFoundException("Trying to move an absent" - + " application " + applicationId); - } - - if (!checkAccess(callerUGI, application.getUser(), - ApplicationAccessType.MODIFY_APP, application)) { - RMAuditLogger.logFailure(callerUGI.getShortUserName(), - AuditConstants.MOVE_APP_REQUEST, - "User doesn't have permissions to " - + ApplicationAccessType.MODIFY_APP.toString(), "ClientRMService", - AuditConstants.UNAUTHORIZED_USER, applicationId); - throw RPCUtil.getRemoteException(new AccessControlException("User " - + callerUGI.getShortUserName() + " cannot perform operation " - + ApplicationAccessType.MODIFY_APP.name() + " on " + applicationId)); - } + UserGroupInformation callerUGI = getCallerUgi(applicationId, + AuditConstants.MOVE_APP_REQUEST); + RMApp application = verifyUserAccessForRMApp(applicationId, callerUGI, + AuditConstants.MOVE_APP_REQUEST, ApplicationAccessType.MODIFY_APP, + true); String targetQueue = request.getTargetQueue(); if (!accessToTargetQueueAllowed(callerUGI, application, targetQueue)) { @@ -1245,9 +1146,8 @@ public class ClientRMService extends AbstractService implements RMAuditLogger.logSuccess(callerUGI.getShortUserName(), AuditConstants.MOVE_APP_REQUEST, "ClientRMService" , applicationId); - MoveApplicationAcrossQueuesResponse response = recordFactory + return recordFactory .newRecordInstance(MoveApplicationAcrossQueuesResponse.class); - return response; } /** @@ -1304,7 +1204,7 @@ public class ClientRMService extends AbstractService implements @Override public GetNewReservationResponse getNewReservation( GetNewReservationRequest request) throws YarnException, IOException { - checkReservationSytem(AuditConstants.CREATE_NEW_RESERVATION_REQUEST); + checkReservationSystem(AuditConstants.CREATE_NEW_RESERVATION_REQUEST); GetNewReservationResponse response = recordFactory.newRecordInstance(GetNewReservationResponse.class); @@ -1318,7 +1218,7 @@ public class ClientRMService extends AbstractService implements public ReservationSubmissionResponse submitReservation( ReservationSubmissionRequest request) throws YarnException, IOException { // Check if reservation system is enabled - checkReservationSytem(AuditConstants.SUBMIT_RESERVATION_REQUEST); + checkReservationSystem(AuditConstants.SUBMIT_RESERVATION_REQUEST); ReservationSubmissionResponse response = recordFactory.newRecordInstance(ReservationSubmissionResponse.class); ReservationId reservationId = request.getReservationId(); @@ -1377,7 +1277,7 @@ public class ClientRMService extends AbstractService implements public ReservationUpdateResponse updateReservation( ReservationUpdateRequest request) throws YarnException, IOException { // Check if reservation system is enabled - checkReservationSytem(AuditConstants.UPDATE_RESERVATION_REQUEST); + checkReservationSystem(AuditConstants.UPDATE_RESERVATION_REQUEST); ReservationUpdateResponse response = recordFactory.newRecordInstance(ReservationUpdateResponse.class); // Validate the input @@ -1416,7 +1316,7 @@ public class ClientRMService extends AbstractService implements public ReservationDeleteResponse deleteReservation( ReservationDeleteRequest request) throws YarnException, IOException { // Check if reservation system is enabled - checkReservationSytem(AuditConstants.DELETE_RESERVATION_REQUEST); + checkReservationSystem(AuditConstants.DELETE_RESERVATION_REQUEST); ReservationDeleteResponse response = recordFactory.newRecordInstance(ReservationDeleteResponse.class); // Validate the input @@ -1455,7 +1355,7 @@ public class ClientRMService extends AbstractService implements public ReservationListResponse listReservations( ReservationListRequest requestInfo) throws YarnException, IOException { // Check if reservation system is enabled - checkReservationSytem(AuditConstants.LIST_RESERVATION_REQUEST); + checkReservationSystem(AuditConstants.LIST_RESERVATION_REQUEST); ReservationListResponse response = recordFactory.newRecordInstance(ReservationListResponse.class); @@ -1495,9 +1395,7 @@ public class ClientRMService extends AbstractService implements public GetNodesToLabelsResponse getNodeToLabels( GetNodesToLabelsRequest request) throws YarnException, IOException { RMNodeLabelsManager labelsMgr = rmContext.getNodeLabelManager(); - GetNodesToLabelsResponse response = - GetNodesToLabelsResponse.newInstance(labelsMgr.getNodeLabels()); - return response; + return GetNodesToLabelsResponse.newInstance(labelsMgr.getNodeLabels()); } @Override @@ -1505,8 +1403,7 @@ public class ClientRMService extends AbstractService implements GetLabelsToNodesRequest request) throws YarnException, IOException { RMNodeLabelsManager labelsMgr = rmContext.getNodeLabelManager(); if (request.getNodeLabels() == null || request.getNodeLabels().isEmpty()) { - return GetLabelsToNodesResponse.newInstance( - labelsMgr.getLabelsToNodes()); + return GetLabelsToNodesResponse.newInstance(labelsMgr.getLabelsToNodes()); } else { return GetLabelsToNodesResponse.newInstance( labelsMgr.getLabelsToNodes(request.getNodeLabels())); @@ -1517,13 +1414,12 @@ public class ClientRMService extends AbstractService implements public GetClusterNodeLabelsResponse getClusterNodeLabels( GetClusterNodeLabelsRequest request) throws YarnException, IOException { RMNodeLabelsManager labelsMgr = rmContext.getNodeLabelManager(); - GetClusterNodeLabelsResponse response = - GetClusterNodeLabelsResponse.newInstance( + return GetClusterNodeLabelsResponse.newInstance( labelsMgr.getClusterNodeLabels()); - return response; } - private void checkReservationSytem(String auditConstant) throws YarnException { + private void checkReservationSystem(String auditConstant) + throws YarnException { // Check if reservation is enabled if (reservationSystem == null) { throw RPCUtil.getRemoteException("Reservation is not enabled." @@ -1642,7 +1538,8 @@ public class ClientRMService extends AbstractService implements UserGroupInformation callerUGI = getCallerUgi(applicationId, AuditConstants.UPDATE_APP_PRIORITY); RMApp application = verifyUserAccessForRMApp(applicationId, callerUGI, - AuditConstants.UPDATE_APP_PRIORITY); + AuditConstants.UPDATE_APP_PRIORITY, ApplicationAccessType.MODIFY_APP, + true); UpdateApplicationPriorityResponse response = recordFactory .newRecordInstance(UpdateApplicationPriorityResponse.class); @@ -1685,9 +1582,14 @@ public class ClientRMService extends AbstractService implements } /** - * Signal a container. + * Send a signal to a container. + * * After the request passes some sanity check, it will be delivered * to RMNodeImpl so that the next NM heartbeat will pick up the signal request + * @param request request to signal a container + * @return the response of sending signal request + * @throws YarnException rpc related exception + * @throws IOException fail to obtain user group information */ @SuppressWarnings("unchecked") @Override @@ -1759,7 +1661,8 @@ public class ClientRMService extends AbstractService implements UserGroupInformation callerUGI = getCallerUgi(applicationId, AuditConstants.UPDATE_APP_TIMEOUTS); RMApp application = verifyUserAccessForRMApp(applicationId, callerUGI, - AuditConstants.UPDATE_APP_TIMEOUTS); + AuditConstants.UPDATE_APP_TIMEOUTS, ApplicationAccessType.MODIFY_APP, + true); if (applicationTimeouts.isEmpty()) { String message = @@ -1778,7 +1681,7 @@ public class ClientRMService extends AbstractService implements if (!EnumSet .of(RMAppState.SUBMITTED, RMAppState.ACCEPTED, RMAppState.RUNNING) .contains(state)) { - if (COMPLETED_APP_STATES.contains(state)) { + if (application.isAppInCompletedStates()) { // If Application is in any of the final states, update timeout // can be skipped rather throwing exception. RMAuditLogger.logSuccess(callerUGI.getShortUserName(), @@ -1823,26 +1726,35 @@ public class ClientRMService extends AbstractService implements } private RMApp verifyUserAccessForRMApp(ApplicationId applicationId, - UserGroupInformation callerUGI, String operation) throws YarnException { + UserGroupInformation callerUGI, String operation, + ApplicationAccessType accessType, + boolean needCheckAccess) throws YarnException { RMApp application = this.rmContext.getRMApps().get(applicationId); if (application == null) { RMAuditLogger.logFailure(callerUGI.getUserName(), operation, "UNKNOWN", "ClientRMService", "Trying to " + operation + " of an absent application", applicationId); - throw new ApplicationNotFoundException("Trying to " + operation - + " of an absent application " + applicationId); + // If the RM doesn't have the application, throw + // ApplicationNotFoundException and let client to handle. + throw new ApplicationNotFoundException("Application with id '" + + applicationId + "' doesn't exist in RM. " + + "Please check that the job " + + "submission was successful."); } - if (!checkAccess(callerUGI, application.getUser(), - ApplicationAccessType.MODIFY_APP, application)) { - RMAuditLogger.logFailure(callerUGI.getShortUserName(), operation, - "User doesn't have permissions to " - + ApplicationAccessType.MODIFY_APP.toString(), - "ClientRMService", AuditConstants.UNAUTHORIZED_USER, applicationId); - throw RPCUtil.getRemoteException(new AccessControlException("User " - + callerUGI.getShortUserName() + " cannot perform operation " - + ApplicationAccessType.MODIFY_APP.name() + " on " + applicationId)); + if (needCheckAccess) { + if (!checkAccess(callerUGI, application.getUser(), + accessType, application)) { + RMAuditLogger.logFailure(callerUGI.getShortUserName(), operation, + "User doesn't have permissions to " + + accessType.toString(), + "ClientRMService", AuditConstants.UNAUTHORIZED_USER, + applicationId); + throw RPCUtil.getRemoteException(new AccessControlException("User " + + callerUGI.getShortUserName() + " cannot perform operation " + + accessType.name() + " on " + applicationId)); + } } return application; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java index 051d979e12e..fc0b2655802 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java @@ -54,6 +54,11 @@ public class RMAuditLogger { public static final String GET_APP_STATE = "Get Application State"; public static final String GET_APP_PRIORITY = "Get Application Priority"; public static final String GET_APP_QUEUE = "Get Application Queue"; + public static final String GET_APP_ATTEMPTS = "Get Application Attempts"; + public static final String GET_APP_ATTEMPT_REPORT + = "Get Application Attempt Report"; + public static final String GET_CONTAINERS = "Get Containers"; + public static final String GET_CONTAINER_REPORT = "Get Container Report"; public static final String FINISH_SUCCESS_APP = "Application Finished - Succeeded"; public static final String FINISH_FAILED_APP = "Application Finished - Failed"; public static final String FINISH_KILLED_APP = "Application Finished - Killed"; From d8e6d6df4c23aa11455c1775391b1c7883bec2ea Mon Sep 17 00:00:00 2001 From: Junping Du Date: Sat, 25 Mar 2017 14:02:25 -0700 Subject: [PATCH 131/188] Add 2.8.0 RELEASENOTEs.md and CHANGES.md. --- .../markdown/release/2.8.0/CHANGES.2.8.0.md | 2986 +++++++++++++++++ .../release/2.8.0/RELEASENOTES.2.8.0.md | 1105 ++++++ 2 files changed, 4091 insertions(+) create mode 100644 hadoop-common-project/hadoop-common/src/site/markdown/release/2.8.0/CHANGES.2.8.0.md create mode 100644 hadoop-common-project/hadoop-common/src/site/markdown/release/2.8.0/RELEASENOTES.2.8.0.md diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/2.8.0/CHANGES.2.8.0.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/2.8.0/CHANGES.2.8.0.md new file mode 100644 index 00000000000..7f9bff26b99 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/2.8.0/CHANGES.2.8.0.md @@ -0,0 +1,2986 @@ + + +# "Apache Hadoop" Changelog + +## Release 2.8.0 - 2017-03-17 + +### INCOMPATIBLE CHANGES: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [YARN-3241](https://issues.apache.org/jira/browse/YARN-3241) | FairScheduler handles "invalid" queue names inconsistently | Major | fairscheduler | zhihai xu | zhihai xu | +| [HADOOP-11731](https://issues.apache.org/jira/browse/HADOOP-11731) | Rework the changelog and releasenotes | Major | documentation | Allen Wittenauer | Allen Wittenauer | +| [HADOOP-11746](https://issues.apache.org/jira/browse/HADOOP-11746) | rewrite test-patch.sh | Major | build, test | Allen Wittenauer | Allen Wittenauer | +| [HDFS-8226](https://issues.apache.org/jira/browse/HDFS-8226) | Non-HA rollback compatibility broken | Blocker | . | J.Andreina | J.Andreina | +| [HADOOP-11772](https://issues.apache.org/jira/browse/HADOOP-11772) | RPC Invoker relies on static ClientCache which has synchronized(this) blocks | Major | ipc, performance | Gopal V | Haohui Mai | +| [YARN-2336](https://issues.apache.org/jira/browse/YARN-2336) | Fair scheduler REST api returns a missing '[' bracket JSON for deep queue tree | Major | fairscheduler | Kenji Kikushima | Akira Ajisaka | +| [YARN-41](https://issues.apache.org/jira/browse/YARN-41) | The RM should handle the graceful shutdown of the NM. | Major | nodemanager, resourcemanager | Ravi Teja Ch N V | Devaraj K | +| [HDFS-6564](https://issues.apache.org/jira/browse/HDFS-6564) | Use slf4j instead of common-logging in hdfs-client | Major | build | Haohui Mai | Rakesh R | +| [MAPREDUCE-6427](https://issues.apache.org/jira/browse/MAPREDUCE-6427) | Fix typo in JobHistoryEventHandler | Minor | . | Brahma Reddy Battula | Ray Chiang | +| [HADOOP-12269](https://issues.apache.org/jira/browse/HADOOP-12269) | Update aws-sdk dependency to 1.10.6; move to aws-sdk-s3 | Major | fs/s3 | Thomas Demoor | Thomas Demoor | +| [HDFS-8900](https://issues.apache.org/jira/browse/HDFS-8900) | Compact XAttrs to optimize memory footprint. | Major | namenode | Yi Liu | Yi Liu | +| [YARN-4087](https://issues.apache.org/jira/browse/YARN-4087) | Followup fixes after YARN-2019 regarding RM behavior when state-store error occurs | Major | . | Jian He | Jian He | +| [HADOOP-12416](https://issues.apache.org/jira/browse/HADOOP-12416) | Trash messages should be handled by Logger instead of being delivered on System.out | Major | trash | Ashutosh Chauhan | Mingliang Liu | +| [HDFS-9063](https://issues.apache.org/jira/browse/HDFS-9063) | Correctly handle snapshot path for getContentSummary | Major | namenode | Jing Zhao | Jing Zhao | +| [HDFS-9433](https://issues.apache.org/jira/browse/HDFS-9433) | DFS getEZForPath API on a non-existent file should throw FileNotFoundException | Major | encryption | Rakesh R | Rakesh R | +| [HADOOP-11252](https://issues.apache.org/jira/browse/HADOOP-11252) | RPC client does not time out by default | Critical | ipc | Wilfred Spiegelenburg | Masatake Iwasaki | +| [HDFS-9047](https://issues.apache.org/jira/browse/HDFS-9047) | Retire libwebhdfs | Major | webhdfs | Allen Wittenauer | Haohui Mai | +| [HADOOP-12651](https://issues.apache.org/jira/browse/HADOOP-12651) | Replace dev-support with wrappers to Yetus | Major | scripts | Allen Wittenauer | Allen Wittenauer | +| [HADOOP-12552](https://issues.apache.org/jira/browse/HADOOP-12552) | Fix undeclared/unused dependency to httpclient | Minor | build | Masatake Iwasaki | Masatake Iwasaki | +| [HADOOP-11792](https://issues.apache.org/jira/browse/HADOOP-11792) | Remove all of the CHANGES.txt files | Major | build | Allen Wittenauer | Andrew Wang | +| [YARN-5035](https://issues.apache.org/jira/browse/YARN-5035) | FairScheduler: Adjust maxAssign dynamically when assignMultiple is turned on | Major | fairscheduler | Karthik Kambatla | Karthik Kambatla | +| [HADOOP-12892](https://issues.apache.org/jira/browse/HADOOP-12892) | fix/rewrite create-release | Blocker | build | Allen Wittenauer | Allen Wittenauer | +| [HADOOP-13139](https://issues.apache.org/jira/browse/HADOOP-13139) | Branch-2: S3a to use thread pool that blocks clients | Major | fs/s3 | Pieter Reuse | Pieter Reuse | +| [HADOOP-13382](https://issues.apache.org/jira/browse/HADOOP-13382) | remove unneeded commons-httpclient dependencies from POM files in Hadoop and sub-projects | Major | build | Matt Foley | Matt Foley | +| [HDFS-7933](https://issues.apache.org/jira/browse/HDFS-7933) | fsck should also report decommissioning replicas. | Major | namenode | Jitendra Nath Pandey | Xiaoyu Yao | +| [HADOOP-13560](https://issues.apache.org/jira/browse/HADOOP-13560) | S3ABlockOutputStream to support huge (many GB) file writes | Major | fs/s3 | Steve Loughran | Steve Loughran | +| [HDFS-11048](https://issues.apache.org/jira/browse/HDFS-11048) | Audit Log should escape control characters | Major | . | Eric Badger | Eric Badger | +| [HADOOP-13812](https://issues.apache.org/jira/browse/HADOOP-13812) | Upgrade Tomcat to 6.0.48 | Blocker | kms | John Zhuge | John Zhuge | +| [HADOOP-13929](https://issues.apache.org/jira/browse/HADOOP-13929) | ADLS connector should not check in contract-test-options.xml | Major | fs/adl, test | John Zhuge | John Zhuge | + + +### IMPORTANT ISSUES: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [YARN-4785](https://issues.apache.org/jira/browse/YARN-4785) | inconsistent value type of the "type" field for LeafQueueInfo in response of RM REST API - cluster/scheduler | Major | webapp | Jayesh | Varun Vasudev | + + +### NEW FEATURES: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HADOOP-8934](https://issues.apache.org/jira/browse/HADOOP-8934) | Shell command ls should include sort options | Minor | fs | Jonathan Allen | Jonathan Allen | +| [HADOOP-9477](https://issues.apache.org/jira/browse/HADOOP-9477) | Add posixGroups support for LDAP groups mapping service | Major | . | Kai Zheng | Dapeng Sun | +| [HDFS-8009](https://issues.apache.org/jira/browse/HDFS-8009) | Signal congestion on the DataNode | Major | datanode | Haohui Mai | Haohui Mai | +| [YARN-2901](https://issues.apache.org/jira/browse/YARN-2901) | Add errors and warning metrics page to RM, NM web UI | Major | nodemanager, resourcemanager | Varun Vasudev | Varun Vasudev | +| [HDFS-7891](https://issues.apache.org/jira/browse/HDFS-7891) | A block placement policy with best rack failure tolerance | Minor | namenode | Walter Su | Walter Su | +| [HADOOP-11843](https://issues.apache.org/jira/browse/HADOOP-11843) | Make setting up the build environment easier | Major | build | Niels Basjes | Niels Basjes | +| [MAPREDUCE-6284](https://issues.apache.org/jira/browse/MAPREDUCE-6284) | Add Task Attempt State API to MapReduce Application Master REST API | Minor | . | Ryu Kobayashi | Ryu Kobayashi | +| [HADOOP-10971](https://issues.apache.org/jira/browse/HADOOP-10971) | Add -C flag to make `hadoop fs -ls` print filenames only | Major | fs | Ryan Williams | Kengo Seki | +| [MAPREDUCE-6364](https://issues.apache.org/jira/browse/MAPREDUCE-6364) | Add a "Kill" link to Task Attempts page | Minor | applicationmaster | Ryu Kobayashi | Ryu Kobayashi | +| [MAPREDUCE-6304](https://issues.apache.org/jira/browse/MAPREDUCE-6304) | Specifying node labels when submitting MR jobs | Major | . | Jian Fang | Naganarasimha G R | +| [HDFS-8487](https://issues.apache.org/jira/browse/HDFS-8487) | Generalize BlockInfo in preparation of merging HDFS-7285 into trunk and branch-2 | Major | namenode | Zhe Zhang | Zhe Zhang | +| [HDFS-8608](https://issues.apache.org/jira/browse/HDFS-8608) | Merge HDFS-7912 to trunk and branch-2 (track BlockInfo instead of Block in UnderReplicatedBlocks and PendingReplicationBlocks) | Major | . | Zhe Zhang | Zhe Zhang | +| [HADOOP-5732](https://issues.apache.org/jira/browse/HADOOP-5732) | Add SFTP FileSystem | Minor | fs | Íñigo Goiri | ramtin | +| [HDFS-8622](https://issues.apache.org/jira/browse/HDFS-8622) | Implement GETCONTENTSUMMARY operation for WebImageViewer | Major | . | Jagadesh Kiran N | Jagadesh Kiran N | +| [HDFS-8155](https://issues.apache.org/jira/browse/HDFS-8155) | Support OAuth2 in WebHDFS | Major | webhdfs | Jakob Homan | Jakob Homan | +| [MAPREDUCE-6415](https://issues.apache.org/jira/browse/MAPREDUCE-6415) | Create a tool to combine aggregated logs into HAR files | Major | . | Robert Kanter | Robert Kanter | +| [HADOOP-12360](https://issues.apache.org/jira/browse/HADOOP-12360) | Create StatsD metrics2 sink | Minor | metrics | Dave Marion | Dave Marion | +| [YARN-261](https://issues.apache.org/jira/browse/YARN-261) | Ability to fail AM attempts | Major | api | Jason Lowe | Rohith Sharma K S | +| [HDFS-9184](https://issues.apache.org/jira/browse/HDFS-9184) | Logging HDFS operation's caller context into audit logs | Major | . | Mingliang Liu | Mingliang Liu | +| [HDFS-9057](https://issues.apache.org/jira/browse/HDFS-9057) | allow/disallow snapshots via webhdfs | Major | webhdfs | Allen Wittenauer | Brahma Reddy Battula | +| [YARN-4349](https://issues.apache.org/jira/browse/YARN-4349) | Support CallerContext in YARN | Major | . | Wangda Tan | Wangda Tan | +| [YARN-3623](https://issues.apache.org/jira/browse/YARN-3623) | We should have a config to indicate the Timeline Service version | Major | timelineserver | Zhijie Shen | Xuan Gong | +| [HADOOP-12657](https://issues.apache.org/jira/browse/HADOOP-12657) | Add a option to skip newline on empty files with getMerge -nl | Minor | . | Jan Filipiak | Kanaka Kumar Avvaru | +| [YARN-3458](https://issues.apache.org/jira/browse/YARN-3458) | CPU resource monitoring in Windows | Minor | nodemanager | Inigo Goiri | Inigo Goiri | +| [HADOOP-12691](https://issues.apache.org/jira/browse/HADOOP-12691) | Add CSRF Filter for REST APIs to Hadoop Common | Major | security | Larry McCay | Larry McCay | +| [HADOOP-12635](https://issues.apache.org/jira/browse/HADOOP-12635) | Adding Append API support for WASB | Major | fs/azure | Dushyanth | Dushyanth | +| [HADOOP-12426](https://issues.apache.org/jira/browse/HADOOP-12426) | Add Entry point for Kerberos health check | Minor | security | Steve Loughran | Steve Loughran | +| [HDFS-9244](https://issues.apache.org/jira/browse/HDFS-9244) | Support nested encryption zones | Major | encryption | Xiaoyu Yao | Zhe Zhang | +| [HADOOP-12548](https://issues.apache.org/jira/browse/HADOOP-12548) | Read s3a creds from a Credential Provider | Major | fs/s3 | Allen Wittenauer | Larry McCay | +| [HDFS-9711](https://issues.apache.org/jira/browse/HDFS-9711) | Integrate CSRF prevention filter in WebHDFS. | Major | datanode, namenode, webhdfs | Chris Nauroth | Chris Nauroth | +| [HDFS-9835](https://issues.apache.org/jira/browse/HDFS-9835) | OIV: add ReverseXML processor which reconstructs an fsimage from an XML file | Major | tools | Colin P. McCabe | Colin P. McCabe | +| [HDFS-9239](https://issues.apache.org/jira/browse/HDFS-9239) | DataNode Lifeline Protocol: an alternative protocol for reporting DataNode liveness | Major | datanode, namenode | Chris Nauroth | Chris Nauroth | +| [HADOOP-12909](https://issues.apache.org/jira/browse/HADOOP-12909) | Change ipc.Client to support asynchronous calls | Major | ipc | Tsz Wo Nicholas Sze | Xiaobing Zhou | +| [HDFS-9945](https://issues.apache.org/jira/browse/HDFS-9945) | Datanode command for evicting writers | Major | datanode | Kihwal Lee | Kihwal Lee | +| [HADOOP-13008](https://issues.apache.org/jira/browse/HADOOP-13008) | Add XFS Filter for UIs to Hadoop Common | Major | security | Larry McCay | Larry McCay | +| [HADOOP-13065](https://issues.apache.org/jira/browse/HADOOP-13065) | Add a new interface for retrieving FS and FC Statistics | Major | fs | Ram Venkatesh | Mingliang Liu | +| [HADOOP-12723](https://issues.apache.org/jira/browse/HADOOP-12723) | S3A: Add ability to plug in any AWSCredentialsProvider | Major | fs/s3 | Steven K. Wong | Steven K. Wong | +| [HADOOP-13226](https://issues.apache.org/jira/browse/HADOOP-13226) | Support async call retry and failover | Major | io, ipc | Xiaobing Zhou | Tsz Wo Nicholas Sze | +| [HADOOP-12537](https://issues.apache.org/jira/browse/HADOOP-12537) | S3A to support Amazon STS temporary credentials | Minor | fs/s3 | Sean Mackrory | Sean Mackrory | +| [HDFS-9804](https://issues.apache.org/jira/browse/HDFS-9804) | Allow long-running Balancer to login with keytab | Major | balancer & mover, security | Xiao Chen | Xiao Chen | +| [HDFS-10918](https://issues.apache.org/jira/browse/HDFS-10918) | Add a tool to get FileEncryptionInfo from CLI | Major | encryption | Xiao Chen | Xiao Chen | +| [HADOOP-13716](https://issues.apache.org/jira/browse/HADOOP-13716) | Add LambdaTestUtils class for tests; fix eventual consistency problem in contract test setup | Major | test | Steve Loughran | Steve Loughran | +| [HADOOP-14049](https://issues.apache.org/jira/browse/HADOOP-14049) | Honour AclBit flag associated to file/folder permission for Azure datalake account | Major | fs/adl | Vishwajeet Dusane | Vishwajeet Dusane | +| [HADOOP-14048](https://issues.apache.org/jira/browse/HADOOP-14048) | REDO operation of WASB#AtomicRename should create placeholder blob for destination folder | Critical | fs/azure | NITIN VERMA | NITIN VERMA | + + +### IMPROVEMENTS: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HDFS-2580](https://issues.apache.org/jira/browse/HDFS-2580) | NameNode#main(...) can make use of GenericOptionsParser. | Minor | namenode | Harsh J | Harsh J | +| [MAPREDUCE-5232](https://issues.apache.org/jira/browse/MAPREDUCE-5232) | log classpath and other key properties on child JVM start | Major | mrv1, mrv2 | Sangjin Lee | Sangjin Lee | +| [HADOOP-7713](https://issues.apache.org/jira/browse/HADOOP-7713) | dfs -count -q should label output column | Trivial | . | Nigel Daley | Jonathan Allen | +| [HDFS-7546](https://issues.apache.org/jira/browse/HDFS-7546) | Document, and set an accepting default for dfs.namenode.kerberos.principal.pattern | Minor | security | Harsh J | Harsh J | +| [HADOOP-11692](https://issues.apache.org/jira/browse/HADOOP-11692) | Improve authentication failure WARN message to avoid user confusion | Major | ipc | Yongjun Zhang | Yongjun Zhang | +| [HADOOP-11226](https://issues.apache.org/jira/browse/HADOOP-11226) | Add a configuration to set ipc.Client's traffic class with IPTOS\_LOWDELAY\|IPTOS\_RELIABILITY | Major | ipc | Gopal V | Gopal V | +| [HADOOP-11711](https://issues.apache.org/jira/browse/HADOOP-11711) | Provide a default value for AES/CTR/NoPadding CryptoCodec classes | Minor | . | Andrew Wang | Andrew Wang | +| [MAPREDUCE-4414](https://issues.apache.org/jira/browse/MAPREDUCE-4414) | Add main methods to JobConf and YarnConfiguration, for debug purposes | Major | client | Harsh J | Plamen Jeliazkov | +| [MAPREDUCE-6105](https://issues.apache.org/jira/browse/MAPREDUCE-6105) | Inconsistent configuration in property mapreduce.reduce.shuffle.merge.percent | Trivial | . | Dongwook Kwon | Ray Chiang | +| [HDFS-2360](https://issues.apache.org/jira/browse/HDFS-2360) | Ugly stacktrace when quota exceeds | Minor | hdfs-client | Rajit Saha | Harsh J | +| [MAPREDUCE-6100](https://issues.apache.org/jira/browse/MAPREDUCE-6100) | replace "mapreduce.job.credentials.binary" with MRJobConfig.MAPREDUCE\_JOB\_CREDENTIALS\_BINARY for better readability. | Trivial | mrv2 | zhihai xu | zhihai xu | +| [MAPREDUCE-5755](https://issues.apache.org/jira/browse/MAPREDUCE-5755) | MapTask.MapOutputBuffer#compare/swap should have @Override annotation | Trivial | . | Tsuyoshi Ozawa | Tsuyoshi Ozawa | +| [MAPREDUCE-4653](https://issues.apache.org/jira/browse/MAPREDUCE-4653) | TestRandomAlgorithm has an unused "import" statement | Trivial | contrib/gridmix | Amir Sanjar | Amir Sanjar | +| [HADOOP-11659](https://issues.apache.org/jira/browse/HADOOP-11659) | o.a.h.fs.FileSystem.Cache#remove should use a single hash map lookup | Minor | fs | Gera Shegalov | Brahma Reddy Battula | +| [HADOOP-11709](https://issues.apache.org/jira/browse/HADOOP-11709) | Time.NANOSECONDS\_PER\_MILLISECOND - use class-level final constant instead of method variable | Trivial | . | Ajith S | Ajith S | +| [HDFS-7835](https://issues.apache.org/jira/browse/HDFS-7835) | make initial sleeptime in locateFollowingBlock configurable for DFSClient. | Major | hdfs-client | zhihai xu | zhihai xu | +| [HDFS-7829](https://issues.apache.org/jira/browse/HDFS-7829) | Code clean up for LocatedBlock | Minor | . | Jing Zhao | Takanobu Asanuma | +| [MAPREDUCE-6282](https://issues.apache.org/jira/browse/MAPREDUCE-6282) | Reuse historyFileAbsolute.getFileSystem in CompletedJob#loadFullHistoryData for code optimization. | Trivial | jobhistoryserver | zhihai xu | zhihai xu | +| [HADOOP-11447](https://issues.apache.org/jira/browse/HADOOP-11447) | Add a more meaningful toString method to SampleStat and MutableStat | Minor | metrics | Karthik Kambatla | Karthik Kambatla | +| [YARN-3350](https://issues.apache.org/jira/browse/YARN-3350) | YARN RackResolver spams logs with messages at info level | Major | . | Wilfred Spiegelenburg | Wilfred Spiegelenburg | +| [MAPREDUCE-6239](https://issues.apache.org/jira/browse/MAPREDUCE-6239) | Consolidate TestJobConf classes in hadoop-mapreduce-client-jobclient and hadoop-mapreduce-client-core | Minor | client | Varun Saxena | Varun Saxena | +| [MAPREDUCE-5190](https://issues.apache.org/jira/browse/MAPREDUCE-5190) | Unnecessary condition test in RandomSampler | Minor | mrv2 | Jingguo Yao | Jingguo Yao | +| [MAPREDUCE-6287](https://issues.apache.org/jira/browse/MAPREDUCE-6287) | Deprecated methods in org.apache.hadoop.examples.Sort | Minor | examples | Chao Zhang | Chao Zhang | +| [HADOOP-11737](https://issues.apache.org/jira/browse/HADOOP-11737) | mockito's version in hadoop-nfs’ pom.xml shouldn't be specified | Minor | nfs | Kengo Seki | Kengo Seki | +| [YARN-2868](https://issues.apache.org/jira/browse/YARN-2868) | FairScheduler: Metric for latency to allocate first container for an application | Major | . | Ray Chiang | Ray Chiang | +| [HDFS-7875](https://issues.apache.org/jira/browse/HDFS-7875) | Improve log message when wrong value configured for dfs.datanode.failed.volumes.tolerated | Trivial | datanode | nijel | nijel | +| [HDFS-7793](https://issues.apache.org/jira/browse/HDFS-7793) | Refactor DFSOutputStream separating DataStreamer out | Major | hdfs-client | Kai Zheng | Li Bo | +| [HADOOP-11741](https://issues.apache.org/jira/browse/HADOOP-11741) | Add LOG.isDebugEnabled() guard for some LOG.debug() | Major | . | Walter Su | Walter Su | +| [MAPREDUCE-579](https://issues.apache.org/jira/browse/MAPREDUCE-579) | Streaming "slowmatch" documentation | Trivial | contrib/streaming | Bo Adler | Harsh J | +| [HDFS-7928](https://issues.apache.org/jira/browse/HDFS-7928) | Scanning blocks from disk during rolling upgrade startup takes a lot of time if disks are busy | Major | datanode | Rushabh S Shah | Rushabh S Shah | +| [HADOOP-11719](https://issues.apache.org/jira/browse/HADOOP-11719) | [Fsshell] Remove bin/hadoop reference from GenericOptionsParser default help text | Minor | scripts | Brahma Reddy Battula | Brahma Reddy Battula | +| [HDFS-8004](https://issues.apache.org/jira/browse/HDFS-8004) | Use KeyProviderCryptoExtension#warmUpEncryptedKeys when creating an encryption zone | Trivial | encryption | Andrew Wang | Andrew Wang | +| [HDFS-7890](https://issues.apache.org/jira/browse/HDFS-7890) | Improve information on Top users for metrics in RollingWindowsManager and lower log level | Major | . | J.Andreina | J.Andreina | +| [HDFS-4396](https://issues.apache.org/jira/browse/HDFS-4396) | Add START\_MSG/SHUTDOWN\_MSG for ZKFC | Major | auto-failover, ha, tools | Liang Xie | Liang Xie | +| [HADOOP-11660](https://issues.apache.org/jira/browse/HADOOP-11660) | Add support for hardware crc of HDFS checksums on ARM aarch64 architecture | Minor | native | Edward Nevill | Edward Nevill | +| [HDFS-7645](https://issues.apache.org/jira/browse/HDFS-7645) | Rolling upgrade is restoring blocks from trash multiple times | Major | datanode | Nathan Roberts | Keisuke Ogiwara | +| [HDFS-3918](https://issues.apache.org/jira/browse/HDFS-3918) | EditLogTailer shouldn't log WARN when other node is in standby mode | Major | ha | Todd Lipcon | Todd Lipcon | +| [HDFS-7944](https://issues.apache.org/jira/browse/HDFS-7944) | Minor cleanup of BlockPoolManager#getAllNamenodeThreads | Minor | . | Arpit Agarwal | Arpit Agarwal | +| [YARN-3258](https://issues.apache.org/jira/browse/YARN-3258) | FairScheduler: Need to add more logging to investigate allocations | Minor | fairscheduler | Anubhav Dhoot | Anubhav Dhoot | +| [HDFS-7671](https://issues.apache.org/jira/browse/HDFS-7671) | hdfs user guide should point to the common rack awareness doc | Major | documentation | Allen Wittenauer | Kai Sasaki | +| [YARN-3412](https://issues.apache.org/jira/browse/YARN-3412) | RM tests should use MockRM where possible | Major | resourcemanager, test | Karthik Kambatla | Karthik Kambatla | +| [YARN-3428](https://issues.apache.org/jira/browse/YARN-3428) | Debug log resources to be localized for a container | Trivial | nodemanager | Karthik Kambatla | Karthik Kambatla | +| [YARN-3424](https://issues.apache.org/jira/browse/YARN-3424) | Change logs for ContainerMonitorImpl's resourse monitoring from info to debug | Major | nodemanager | Anubhav Dhoot | Anubhav Dhoot | +| [YARN-3248](https://issues.apache.org/jira/browse/YARN-3248) | Display count of nodes blacklisted by apps in the web UI | Major | capacityscheduler, resourcemanager | Varun Vasudev | Varun Vasudev | +| [HDFS-7978](https://issues.apache.org/jira/browse/HDFS-7978) | Add LOG.isDebugEnabled() guard for some LOG.debug(..) | Major | . | Walter Su | Walter Su | +| [HDFS-8008](https://issues.apache.org/jira/browse/HDFS-8008) | Support client-side back off when the datanodes are congested | Major | hdfs-client | Haohui Mai | Haohui Mai | +| [HDFS-7888](https://issues.apache.org/jira/browse/HDFS-7888) | Change DataStreamer/DFSOutputStream/DFSPacket for convenience of subclassing | Minor | hdfs-client | Li Bo | Li Bo | +| [HDFS-8035](https://issues.apache.org/jira/browse/HDFS-8035) | Move checkBlocksProperlyReplicated() in FSNamesystem to BlockManager | Minor | namenode | Haohui Mai | Haohui Mai | +| [HADOOP-9805](https://issues.apache.org/jira/browse/HADOOP-9805) | Refactor RawLocalFileSystem#rename for improved testability. | Minor | fs, test | Chris Nauroth | Jean-Pierre Matsumoto | +| [HADOOP-11785](https://issues.apache.org/jira/browse/HADOOP-11785) | Reduce number of listStatus operation in distcp buildListing() | Minor | tools/distcp | Zoran Dimitrijevic | Zoran Dimitrijevic | +| [HADOOP-11717](https://issues.apache.org/jira/browse/HADOOP-11717) | Add Redirecting WebSSO behavior with JWT Token in Hadoop Auth | Major | security | Larry McCay | Larry McCay | +| [YARN-3294](https://issues.apache.org/jira/browse/YARN-3294) | Allow dumping of Capacity Scheduler debug logs via web UI for a fixed time period | Major | capacityscheduler | Varun Vasudev | Varun Vasudev | +| [HDFS-8073](https://issues.apache.org/jira/browse/HDFS-8073) | Split BlockPlacementPolicyDefault.chooseTarget(..) so it can be easily overrided. | Trivial | namenode | Walter Su | Walter Su | +| [HDFS-8076](https://issues.apache.org/jira/browse/HDFS-8076) | Code cleanup for DFSInputStream: use offset instead of LocatedBlock when possible | Major | . | Zhe Zhang | Zhe Zhang | +| [HDFS-7979](https://issues.apache.org/jira/browse/HDFS-7979) | Initialize block report IDs with a random number | Minor | datanode | Andrew Wang | Andrew Wang | +| [HDFS-8101](https://issues.apache.org/jira/browse/HDFS-8101) | DFSClient use of non-constant DFSConfigKeys pulls in WebHDFS classes at runtime | Minor | hdfs-client | Sean Busbey | Sean Busbey | +| [YARN-3293](https://issues.apache.org/jira/browse/YARN-3293) | Track and display capacity scheduler health metrics in web UI | Major | capacityscheduler | Varun Vasudev | Varun Vasudev | +| [MAPREDUCE-6291](https://issues.apache.org/jira/browse/MAPREDUCE-6291) | Correct mapred queue usage command | Minor | client | Brahma Reddy Battula | Brahma Reddy Battula | +| [MAPREDUCE-6307](https://issues.apache.org/jira/browse/MAPREDUCE-6307) | Remove property mapreduce.tasktracker.taskmemorymanager.monitoringinterval | Minor | . | Akira Ajisaka | J.Andreina | +| [YARN-3348](https://issues.apache.org/jira/browse/YARN-3348) | Add a 'yarn top' tool to help understand cluster usage | Major | resourcemanager | Varun Vasudev | Varun Vasudev | +| [YARN-3469](https://issues.apache.org/jira/browse/YARN-3469) | ZKRMStateStore: Avoid setting watches that are not required | Minor | . | Jun Gong | Jun Gong | +| [HDFS-8117](https://issues.apache.org/jira/browse/HDFS-8117) | More accurate verification in SimulatedFSDataset: replace DEFAULT\_DATABYTE with patterned data | Major | . | Zhe Zhang | Zhe Zhang | +| [HDFS-8144](https://issues.apache.org/jira/browse/HDFS-8144) | Split TestLazyPersistFiles into multiple tests | Major | test | Arpit Agarwal | Arpit Agarwal | +| [YARN-3404](https://issues.apache.org/jira/browse/YARN-3404) | View the queue name to YARN Application page | Minor | . | Ryu Kobayashi | Ryu Kobayashi | +| [YARN-3451](https://issues.apache.org/jira/browse/YARN-3451) | Add start time and Elapsed in ApplicationAttemptReport and display the same in RMAttemptBlock WebUI | Major | api, webapp | Rohith Sharma K S | Rohith Sharma K S | +| [HDFS-7863](https://issues.apache.org/jira/browse/HDFS-7863) | Missing description of some methods and parameters in javadoc of FSDirDeleteOp | Minor | . | Yongjun Zhang | Brahma Reddy Battula | +| [HDFS-8152](https://issues.apache.org/jira/browse/HDFS-8152) | Refactoring of lazy persist storage cases | Major | test | Arpit Agarwal | Arpit Agarwal | +| [HDFS-8133](https://issues.apache.org/jira/browse/HDFS-8133) | Improve readability of deleted block check | Major | namenode | Daryn Sharp | Daryn Sharp | +| [HADOOP-11812](https://issues.apache.org/jira/browse/HADOOP-11812) | Implement listLocatedStatus for ViewFileSystem to speed up split calculation | Blocker | fs | Gera Shegalov | Gera Shegalov | +| [MAPREDUCE-6297](https://issues.apache.org/jira/browse/MAPREDUCE-6297) | Task Id of the failed task in diagnostics should link to the task page | Minor | jobhistoryserver | Siqi Li | Siqi Li | +| [HADOOP-11827](https://issues.apache.org/jira/browse/HADOOP-11827) | Speed-up distcp buildListing() using threadpool | Major | tools/distcp | Zoran Dimitrijevic | Zoran Dimitrijevic | +| [YARN-3494](https://issues.apache.org/jira/browse/YARN-3494) | Expose AM resource limit and usage in QueueMetrics | Major | . | Jian He | Rohith Sharma K S | +| [YARN-3503](https://issues.apache.org/jira/browse/YARN-3503) | Expose disk utilization percentage and bad local and log dir counts on NM via JMX | Major | nodemanager | Varun Vasudev | Varun Vasudev | +| [YARN-3410](https://issues.apache.org/jira/browse/YARN-3410) | YARN admin should be able to remove individual application records from RMStateStore | Critical | resourcemanager, yarn | Wangda Tan | Rohith Sharma K S | +| [HDFS-8215](https://issues.apache.org/jira/browse/HDFS-8215) | Refactor NamenodeFsck#check method | Minor | namenode | Takanobu Asanuma | Takanobu Asanuma | +| [YARN-3511](https://issues.apache.org/jira/browse/YARN-3511) | Add errors and warnings page to ATS | Major | timelineserver | Varun Vasudev | Varun Vasudev | +| [HDFS-8176](https://issues.apache.org/jira/browse/HDFS-8176) | Record from/to snapshots in audit log for snapshot diff report | Minor | snapshots | J.Andreina | J.Andreina | +| [YARN-3406](https://issues.apache.org/jira/browse/YARN-3406) | Display count of running containers in the RM's Web UI | Minor | . | Ryu Kobayashi | Ryu Kobayashi | +| [HADOOP-11357](https://issues.apache.org/jira/browse/HADOOP-11357) | Print information of the build enviornment in test-patch.sh | Minor | scripts | Haohui Mai | Allen Wittenauer | +| [HDFS-8204](https://issues.apache.org/jira/browse/HDFS-8204) | Mover/Balancer should not schedule two replicas to the same DN | Minor | balancer & mover | Walter Su | Walter Su | +| [HDFS-8280](https://issues.apache.org/jira/browse/HDFS-8280) | Code Cleanup in DFSInputStream | Minor | . | Jing Zhao | Jing Zhao | +| [HDFS-8283](https://issues.apache.org/jira/browse/HDFS-8283) | DataStreamer cleanup and some minor improvement | Minor | hdfs-client | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze | +| [HDFS-5574](https://issues.apache.org/jira/browse/HDFS-5574) | Remove buffer copy in BlockReader.skip | Trivial | . | Binglin Chang | Binglin Chang | +| [HDFS-7770](https://issues.apache.org/jira/browse/HDFS-7770) | Need document for storage type label of data node storage locations under dfs.datanode.data.dir | Major | documentation | Xiaoyu Yao | Xiaoyu Yao | +| [HDFS-8200](https://issues.apache.org/jira/browse/HDFS-8200) | Refactor FSDirStatAndListingOp | Major | . | Haohui Mai | Haohui Mai | +| [YARN-3363](https://issues.apache.org/jira/browse/YARN-3363) | add localization and container launch time to ContainerMetrics at NM to show these timing information for each active container. | Major | nodemanager | zhihai xu | zhihai xu | +| [HDFS-7397](https://issues.apache.org/jira/browse/HDFS-7397) | Add more detail to the documentation for the conf key "dfs.client.read.shortcircuit.streams.cache.size" | Minor | hdfs-client | Tsz Wo Nicholas Sze | Brahma Reddy Battula | +| [YARN-2980](https://issues.apache.org/jira/browse/YARN-2980) | Move health check script related functionality to hadoop-common | Blocker | . | Ming Ma | Varun Saxena | +| [HADOOP-11911](https://issues.apache.org/jira/browse/HADOOP-11911) | test-patch should allow configuration of default branch | Minor | . | Sean Busbey | Sean Busbey | +| [HDFS-7758](https://issues.apache.org/jira/browse/HDFS-7758) | Retire FsDatasetSpi#getVolumes() and use FsDatasetSpi#getVolumeRefs() instead | Major | datanode | Lei (Eddy) Xu | Lei (Eddy) Xu | +| [HADOOP-11925](https://issues.apache.org/jira/browse/HADOOP-11925) | backport trunk's smart-apply-patch.sh to branch-2 | Major | scripts | Allen Wittenauer | Allen Wittenauer | +| [MAPREDUCE-6192](https://issues.apache.org/jira/browse/MAPREDUCE-6192) | Create unit test to automatically compare MR related classes and mapred-default.xml | Minor | . | Ray Chiang | Ray Chiang | +| [HADOOP-11813](https://issues.apache.org/jira/browse/HADOOP-11813) | releasedocmaker.py should use today's date instead of unreleased | Minor | build | Allen Wittenauer | Darrell Taylor | +| [YARN-3491](https://issues.apache.org/jira/browse/YARN-3491) | PublicLocalizer#addResource is too slow. | Critical | nodemanager | zhihai xu | zhihai xu | +| [MAPREDUCE-6279](https://issues.apache.org/jira/browse/MAPREDUCE-6279) | AM should explicity exit JVM after all services have stopped | Major | . | Jason Lowe | Eric Payne | +| [HDFS-8108](https://issues.apache.org/jira/browse/HDFS-8108) | Fsck should provide the info on mandatory option to be used along with "-blocks , -locations and -racks" | Trivial | documentation | J.Andreina | J.Andreina | +| [HDFS-8207](https://issues.apache.org/jira/browse/HDFS-8207) | Improper log message when blockreport interval compared with initial delay | Minor | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [MAPREDUCE-6079](https://issues.apache.org/jira/browse/MAPREDUCE-6079) | Rename JobImpl#username to reporterUserName | Major | . | Tsuyoshi Ozawa | Tsuyoshi Ozawa | +| [HDFS-8209](https://issues.apache.org/jira/browse/HDFS-8209) | Support different number of datanode directories in MiniDFSCluster. | Minor | test | Surendra Singh Lilhore | Surendra Singh Lilhore | +| [YARN-3169](https://issues.apache.org/jira/browse/YARN-3169) | Drop YARN's overview document | Major | documentation | Allen Wittenauer | Brahma Reddy Battula | +| [YARN-2784](https://issues.apache.org/jira/browse/YARN-2784) | Make POM project names consistent | Minor | build | Rohith Sharma K S | Rohith Sharma K S | +| [MAPREDUCE-5981](https://issues.apache.org/jira/browse/MAPREDUCE-5981) | Log levels of certain MR logs can be changed to DEBUG | Major | . | Varun Saxena | Varun Saxena | +| [HADOOP-6842](https://issues.apache.org/jira/browse/HADOOP-6842) | "hadoop fs -text" does not give a useful text representation of MapWritable objects | Major | io | Steven K. Wong | Akira Ajisaka | +| [YARN-20](https://issues.apache.org/jira/browse/YARN-20) | More information for "yarn.resourcemanager.webapp.address" in yarn-default.xml | Trivial | documentation, resourcemanager | Nemon Lou | Bartosz Ługowski | +| [HDFS-5640](https://issues.apache.org/jira/browse/HDFS-5640) | Add snapshot methods to FileContext. | Major | hdfs-client, snapshots | Chris Nauroth | Rakesh R | +| [HDFS-8284](https://issues.apache.org/jira/browse/HDFS-8284) | Update documentation about how to use HTrace with HDFS | Major | documentation | Masatake Iwasaki | Masatake Iwasaki | +| [HDFS-7433](https://issues.apache.org/jira/browse/HDFS-7433) | Optimize performance of DatanodeManager's node map | Critical | namenode | Daryn Sharp | Daryn Sharp | +| [MAPREDUCE-5248](https://issues.apache.org/jira/browse/MAPREDUCE-5248) | Let NNBenchWithoutMR specify the replication factor for its test | Minor | client, test | Erik Paulson | Erik Paulson | +| [HADOOP-9737](https://issues.apache.org/jira/browse/HADOOP-9737) | JarFinder#getJar should delete the jar file upon destruction of the JVM | Major | util | Esteban Gutierrez | Jean-Baptiste Onofré | +| [YARN-1050](https://issues.apache.org/jira/browse/YARN-1050) | Document the Fair Scheduler REST API | Major | documentation, fairscheduler | Sandy Ryza | Kenji Kikushima | +| [YARN-3271](https://issues.apache.org/jira/browse/YARN-3271) | FairScheduler: Move tests related to max-runnable-apps from TestFairScheduler to TestAppRunnability | Major | . | Karthik Kambatla | nijel | +| [YARN-2206](https://issues.apache.org/jira/browse/YARN-2206) | Update document for applications REST API response examples | Minor | documentation | Kenji Kikushima | Brahma Reddy Battula | +| [HDFS-6757](https://issues.apache.org/jira/browse/HDFS-6757) | Simplify lease manager with INodeID | Major | . | Haohui Mai | Haohui Mai | +| [HDFS-8327](https://issues.apache.org/jira/browse/HDFS-8327) | Simplify quota calculations for snapshots and truncate | Major | . | Haohui Mai | Haohui Mai | +| [YARN-1287](https://issues.apache.org/jira/browse/YARN-1287) | Consolidate MockClocks | Major | . | Sandy Ryza | Sebastian Wong | +| [HDFS-8357](https://issues.apache.org/jira/browse/HDFS-8357) | Consolidate parameters of INode.CleanSubtree() into a parameter objects. | Major | . | Haohui Mai | Li Lu | +| [HADOOP-11950](https://issues.apache.org/jira/browse/HADOOP-11950) | Add cli option to test-patch to set the project-under-test | Minor | . | Sean Busbey | Sean Busbey | +| [HADOOP-11948](https://issues.apache.org/jira/browse/HADOOP-11948) | test-patch's issue matching regex should be configurable. | Major | . | Sean Busbey | Sean Busbey | +| [MAPREDUCE-5465](https://issues.apache.org/jira/browse/MAPREDUCE-5465) | Tasks are often killed before they exit on their own | Major | mr-am, mrv2 | Radim Kolar | Ming Ma | +| [YARN-3513](https://issues.apache.org/jira/browse/YARN-3513) | Remove unused variables in ContainersMonitorImpl and add debug log for overall resource usage by all containers | Trivial | nodemanager | Naganarasimha G R | Naganarasimha G R | +| [HDFS-8255](https://issues.apache.org/jira/browse/HDFS-8255) | Rename getBlockReplication to getPreferredBlockReplication | Major | . | Zhe Zhang | Zhe Zhang | +| [YARN-3613](https://issues.apache.org/jira/browse/YARN-3613) | TestContainerManagerSecurity should init and start Yarn cluster in setup instead of individual methods | Minor | test | Karthik Kambatla | nijel | +| [HDFS-6184](https://issues.apache.org/jira/browse/HDFS-6184) | Capture NN's thread dump when it fails over | Major | namenode | Ming Ma | Ming Ma | +| [YARN-3539](https://issues.apache.org/jira/browse/YARN-3539) | Compatibility doc to state that ATS v1 is a stable REST API | Major | documentation | Steve Loughran | Steve Loughran | +| [HADOOP-9723](https://issues.apache.org/jira/browse/HADOOP-9723) | Improve error message when hadoop archive output path already exists | Trivial | . | Stephen Chu | Yongjun Zhang | +| [HADOOP-11713](https://issues.apache.org/jira/browse/HADOOP-11713) | ViewFileSystem should support snapshot methods. | Major | fs | Chris Nauroth | Rakesh R | +| [HDFS-8350](https://issues.apache.org/jira/browse/HDFS-8350) | Remove old webhdfs.xml and other outdated documentation stuff | Major | documentation | Akira Ajisaka | Brahma Reddy Battula | +| [HADOOP-11960](https://issues.apache.org/jira/browse/HADOOP-11960) | Enable Azure-Storage Client Side logging. | Major | tools | Dushyanth | Dushyanth | +| [HDFS-6888](https://issues.apache.org/jira/browse/HDFS-6888) | Allow selectively audit logging ops | Major | . | Kihwal Lee | Chen He | +| [HDFS-8397](https://issues.apache.org/jira/browse/HDFS-8397) | Refactor the error handling code in DataStreamer | Minor | hdfs-client | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze | +| [HDFS-8394](https://issues.apache.org/jira/browse/HDFS-8394) | Move getAdditionalBlock() and related functionalities into a separate class | Major | . | Haohui Mai | Haohui Mai | +| [HADOOP-11939](https://issues.apache.org/jira/browse/HADOOP-11939) | Deprecate DistCpV1 and Logalyzer | Major | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [HDFS-4185](https://issues.apache.org/jira/browse/HDFS-4185) | Add a metric for number of active leases | Major | namenode | Kihwal Lee | Rakesh R | +| [HADOOP-1540](https://issues.apache.org/jira/browse/HADOOP-1540) | Support file exclusion list in distcp | Minor | util | Senthil Subramanian | Rich Haase | +| [HADOOP-11103](https://issues.apache.org/jira/browse/HADOOP-11103) | Clean up RemoteException | Trivial | ipc | Sean Busbey | Sean Busbey | +| [HDFS-8131](https://issues.apache.org/jira/browse/HDFS-8131) | Implement a space balanced block placement policy | Minor | namenode | Liu Shaohui | Liu Shaohui | +| [HADOOP-11970](https://issues.apache.org/jira/browse/HADOOP-11970) | Replace uses of ThreadLocal\ with JDK7 ThreadLocalRandom | Major | . | Sean Busbey | Sean Busbey | +| [HADOOP-11995](https://issues.apache.org/jira/browse/HADOOP-11995) | Make jetty version configurable from the maven command line | Trivial | build, ld | sriharsha devineni | sriharsha devineni | +| [HDFS-4383](https://issues.apache.org/jira/browse/HDFS-4383) | Document the lease limits | Minor | . | Eli Collins | Mohammad Arshad | +| [HADOOP-10366](https://issues.apache.org/jira/browse/HADOOP-10366) | Add whitespaces between the classes for values in core-default.xml to fit better in browser | Minor | documentation | Chengwei Yang | Kanaka Kumar Avvaru | +| [HADOOP-11594](https://issues.apache.org/jira/browse/HADOOP-11594) | Improve the readability of site index of documentation | Minor | documentation | Masatake Iwasaki | Masatake Iwasaki | +| [YARN-160](https://issues.apache.org/jira/browse/YARN-160) | nodemanagers should obtain cpu/memory values from underlying OS | Major | nodemanager | Alejandro Abdelnur | Varun Vasudev | +| [HADOOP-11242](https://issues.apache.org/jira/browse/HADOOP-11242) | Record the time of calling in tracing span of IPC server | Minor | ipc | Masatake Iwasaki | Masatake Iwasaki | +| [YARN-3722](https://issues.apache.org/jira/browse/YARN-3722) | Merge multiple TestWebAppUtils into o.a.h.yarn.webapp.util.TestWebAppUtils | Minor | test | Masatake Iwasaki | Masatake Iwasaki | +| [HADOOP-11894](https://issues.apache.org/jira/browse/HADOOP-11894) | Bump the version of Apache HTrace to 3.2.0-incubating | Major | . | Masatake Iwasaki | Masatake Iwasaki | +| [YARN-3489](https://issues.apache.org/jira/browse/YARN-3489) | RMServerUtils.validateResourceRequests should only obtain queue info once | Major | resourcemanager | Jason Lowe | Varun Saxena | +| [HDFS-8443](https://issues.apache.org/jira/browse/HDFS-8443) | Document dfs.namenode.service.handler.count in hdfs-site.xml | Major | documentation | Akira Ajisaka | J.Andreina | +| [YARN-3547](https://issues.apache.org/jira/browse/YARN-3547) | FairScheduler: Apps that have no resource demand should not participate scheduling | Major | fairscheduler | Xianyin Xin | Xianyin Xin | +| [YARN-3713](https://issues.apache.org/jira/browse/YARN-3713) | Remove duplicate function call storeContainerDiagnostics in ContainerDiagnosticsUpdateTransition | Minor | nodemanager | zhihai xu | zhihai xu | +| [HADOOP-12043](https://issues.apache.org/jira/browse/HADOOP-12043) | Display warning if defaultFs is not set when running fs commands. | Minor | fs | Lei (Eddy) Xu | Lei (Eddy) Xu | +| [YARN-3467](https://issues.apache.org/jira/browse/YARN-3467) | Expose allocatedMB, allocatedVCores, and runningContainers metrics on running Applications in RM Web UI | Minor | webapp, yarn | Anthony Rojas | Anubhav Dhoot | +| [HDFS-8490](https://issues.apache.org/jira/browse/HDFS-8490) | Typo in trace enabled log in ExceptionHandler of WebHDFS | Trivial | webhdfs | Jakob Homan | Archana T | +| [HDFS-8521](https://issues.apache.org/jira/browse/HDFS-8521) | Add @VisibleForTesting annotation to {{BlockPoolSlice#selectReplicaToDelete}} | Trivial | . | Colin P. McCabe | Colin P. McCabe | +| [MAPREDUCE-6174](https://issues.apache.org/jira/browse/MAPREDUCE-6174) | Combine common stream code into parent class for InMemoryMapOutput and OnDiskMapOutput. | Major | mrv2 | Eric Payne | Eric Payne | +| [HDFS-8532](https://issues.apache.org/jira/browse/HDFS-8532) | Make the visibility of DFSOutputStream#streamer member variable to private | Trivial | . | Rakesh R | Rakesh R | +| [HDFS-8535](https://issues.apache.org/jira/browse/HDFS-8535) | Clarify that dfs usage in dfsadmin -report output includes all block replicas. | Minor | documentation | Lei (Eddy) Xu | Lei (Eddy) Xu | +| [MAPREDUCE-6383](https://issues.apache.org/jira/browse/MAPREDUCE-6383) | Pi job (QuasiMonteCarlo) should not try to read the results file if its job fails | Major | examples | Harsh J | Harsh J | +| [YARN-3259](https://issues.apache.org/jira/browse/YARN-3259) | FairScheduler: Trigger fairShare updates on node events | Major | fairscheduler | Anubhav Dhoot | Anubhav Dhoot | +| [HADOOP-12059](https://issues.apache.org/jira/browse/HADOOP-12059) | S3Credentials should support use of CredentialProvider | Major | fs/s3 | Sean Busbey | Sean Busbey | +| [MAPREDUCE-6354](https://issues.apache.org/jira/browse/MAPREDUCE-6354) | ShuffleHandler should be able to log shuffle connections | Major | . | Chang Li | Chang Li | +| [HADOOP-12055](https://issues.apache.org/jira/browse/HADOOP-12055) | Deprecate usage of NativeIO#link | Major | native | Andrew Wang | Andrew Wang | +| [HDFS-8432](https://issues.apache.org/jira/browse/HDFS-8432) | Introduce a minimum compatible layout version to allow downgrade in more rolling upgrade use cases. | Major | namenode, rolling upgrades | Chris Nauroth | Chris Nauroth | +| [HDFS-8116](https://issues.apache.org/jira/browse/HDFS-8116) | Cleanup uncessary if LOG.isDebugEnabled() from RollingWindowManager | Trivial | namenode | Xiaoyu Yao | Brahma Reddy Battula | +| [YARN-2716](https://issues.apache.org/jira/browse/YARN-2716) | Refactor ZKRMStateStore retry code with Apache Curator | Major | . | Jian He | Karthik Kambatla | +| [HDFS-8553](https://issues.apache.org/jira/browse/HDFS-8553) | Document hdfs class path options | Major | documentation | Xiaoyu Yao | Brahma Reddy Battula | +| [YARN-3786](https://issues.apache.org/jira/browse/YARN-3786) | Document yarn class path options | Major | documentation | Brahma Reddy Battula | Brahma Reddy Battula | +| [MAPREDUCE-6392](https://issues.apache.org/jira/browse/MAPREDUCE-6392) | Document mapred class path options | Major | documentation | Brahma Reddy Battula | Brahma Reddy Battula | +| [HDFS-8549](https://issues.apache.org/jira/browse/HDFS-8549) | Abort the balancer if an upgrade is in progress | Major | balancer & mover | Andrew Wang | Andrew Wang | +| [HDFS-8573](https://issues.apache.org/jira/browse/HDFS-8573) | Move creation of restartMeta file logic from BlockReceiver to ReplicaInPipeline | Major | datanode | Lei (Eddy) Xu | Lei (Eddy) Xu | +| [HADOOP-11971](https://issues.apache.org/jira/browse/HADOOP-11971) | Move test utilities for tracing from hadoop-hdfs to hadoop-common | Minor | tracing | Masatake Iwasaki | Masatake Iwasaki | +| [HDFS-8361](https://issues.apache.org/jira/browse/HDFS-8361) | Choose SSD over DISK in block placement | Major | namenode | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze | +| [YARN-3789](https://issues.apache.org/jira/browse/YARN-3789) | Improve logs for LeafQueue#activateApplications() | Minor | resourcemanager | Bibin A Chundatt | Bibin A Chundatt | +| [HDFS-8606](https://issues.apache.org/jira/browse/HDFS-8606) | Cleanup DFSOutputStream by removing unwanted changes | Minor | hdfs-client | Rakesh R | Rakesh R | +| [YARN-3148](https://issues.apache.org/jira/browse/YARN-3148) | Allow CORS related headers to passthrough in WebAppProxyServlet | Major | . | Prakash Ramachandran | Varun Saxena | +| [HDFS-8589](https://issues.apache.org/jira/browse/HDFS-8589) | Fix unused imports in BPServiceActor and BlockReportLeaseManager | Trivial | . | Colin P. McCabe | Colin P. McCabe | +| [HADOOP-7139](https://issues.apache.org/jira/browse/HADOOP-7139) | Allow appending to existing SequenceFiles | Major | io | Stephen Rose | Kanaka Kumar Avvaru | +| [HDFS-8605](https://issues.apache.org/jira/browse/HDFS-8605) | Merge Refactor of DFSOutputStream from HDFS-7285 branch | Major | . | Vinayakumar B | Vinayakumar B | +| [MAPREDUCE-6395](https://issues.apache.org/jira/browse/MAPREDUCE-6395) | Improve the commit failure messages in MRAppMaster recovery | Major | applicationmaster | Gera Shegalov | Brahma Reddy Battula | +| [HDFS-8582](https://issues.apache.org/jira/browse/HDFS-8582) | Support getting a list of reconfigurable config properties and do not generate spurious reconfig warnings | Minor | datanode, hdfs-client | Lei (Eddy) Xu | Lei (Eddy) Xu | +| [MAPREDUCE-6316](https://issues.apache.org/jira/browse/MAPREDUCE-6316) | Task Attempt List entries should link to the task overview | Major | . | Siqi Li | Siqi Li | +| [MAPREDUCE-6305](https://issues.apache.org/jira/browse/MAPREDUCE-6305) | AM/Task log page should be able to link back to the job | Major | . | Siqi Li | Siqi Li | +| [YARN-3834](https://issues.apache.org/jira/browse/YARN-3834) | Scrub debug logging of tokens during resource localization. | Major | nodemanager | Chris Nauroth | Chris Nauroth | +| [MAPREDUCE-6408](https://issues.apache.org/jira/browse/MAPREDUCE-6408) | Queue name and user name should be printed on the job page | Major | applicationmaster | Siqi Li | Siqi Li | +| [HDFS-8639](https://issues.apache.org/jira/browse/HDFS-8639) | Option for HTTP port of NameNode by MiniDFSClusterManager | Minor | test | Kai Sasaki | Kai Sasaki | +| [YARN-3360](https://issues.apache.org/jira/browse/YARN-3360) | Add JMX metrics to TimelineDataManager | Major | timelineserver | Jason Lowe | Jason Lowe | +| [HADOOP-12049](https://issues.apache.org/jira/browse/HADOOP-12049) | Control http authentication cookie persistence via configuration | Major | security | Benoy Antony | Huizhi Lu | +| [HDFS-8462](https://issues.apache.org/jira/browse/HDFS-8462) | Implement GETXATTRS and LISTXATTRS operations for WebImageViewer | Major | . | Akira Ajisaka | Jagadesh Kiran N | +| [HDFS-8640](https://issues.apache.org/jira/browse/HDFS-8640) | Make reserved RBW space visible through JMX | Major | . | Kanaka Kumar Avvaru | Kanaka Kumar Avvaru | +| [HDFS-8546](https://issues.apache.org/jira/browse/HDFS-8546) | Use try with resources in DataStorage and Storage | Minor | datanode | Andrew Wang | Andrew Wang | +| [HADOOP-11807](https://issues.apache.org/jira/browse/HADOOP-11807) | add a lint mode to releasedocmaker | Minor | build, documentation, yetus | Allen Wittenauer | ramtin | +| [HDFS-8653](https://issues.apache.org/jira/browse/HDFS-8653) | Code cleanup for DatanodeManager, DatanodeDescriptor and DatanodeStorageInfo | Major | . | Zhe Zhang | Zhe Zhang | +| [HDFS-8659](https://issues.apache.org/jira/browse/HDFS-8659) | Block scanner INFO message is spamming logs | Major | datanode | Yongjun Zhang | Yongjun Zhang | +| [MAPREDUCE-6384](https://issues.apache.org/jira/browse/MAPREDUCE-6384) | Add the last reporting reducer info for too many fetch failure diagnostics | Major | . | Chang Li | Chang Li | +| [HADOOP-12158](https://issues.apache.org/jira/browse/HADOOP-12158) | Improve error message in TestCryptoStreamsWithOpensslAesCtrCryptoCodec when OpenSSL is not installed | Trivial | test | Andrew Wang | Andrew Wang | +| [HADOOP-12172](https://issues.apache.org/jira/browse/HADOOP-12172) | FsShell mkdir -p makes an unnecessary check for the existence of the parent. | Minor | fs | Chris Nauroth | Chris Nauroth | +| [HDFS-8703](https://issues.apache.org/jira/browse/HDFS-8703) | Merge refactor of DFSInputStream from ErasureCoding branch | Major | . | Vinayakumar B | Vinayakumar B | +| [HDFS-8709](https://issues.apache.org/jira/browse/HDFS-8709) | Clarify automatic sync in FSEditLog#logEdit | Minor | . | Andrew Wang | Andrew Wang | +| [HADOOP-12045](https://issues.apache.org/jira/browse/HADOOP-12045) | Enable LocalFileSystem#setTimes to change atime | Minor | fs | Kazuho Fujii | Kazuho Fujii | +| [HADOOP-12185](https://issues.apache.org/jira/browse/HADOOP-12185) | NetworkTopology is not efficient adding/getting/removing nodes | Major | . | Inigo Goiri | Inigo Goiri | +| [HADOOP-12135](https://issues.apache.org/jira/browse/HADOOP-12135) | cleanup releasedocmaker | Major | yetus | Allen Wittenauer | Allen Wittenauer | +| [HADOOP-12195](https://issues.apache.org/jira/browse/HADOOP-12195) | Add annotation to package-info.java file to workaround MCOMPILER-205 | Trivial | . | Andrew Wang | Andrew Wang | +| [HADOOP-12193](https://issues.apache.org/jira/browse/HADOOP-12193) | Rename Touchz.java to Touch.java | Trivial | . | Andrew Wang | Andrew Wang | +| [HDFS-8711](https://issues.apache.org/jira/browse/HDFS-8711) | setSpaceQuota command should print the available storage type when input storage type is wrong | Major | hdfs-client | Surendra Singh Lilhore | Surendra Singh Lilhore | +| [HDFS-8712](https://issues.apache.org/jira/browse/HDFS-8712) | Remove "public" and "abstract" modifiers in FsVolumeSpi and FsDatasetSpi | Trivial | . | Lei (Eddy) Xu | Lei (Eddy) Xu | +| [HADOOP-12194](https://issues.apache.org/jira/browse/HADOOP-12194) | Support for incremental generation in the protoc plugin | Major | . | Andrew Wang | Andrew Wang | +| [HADOOP-12180](https://issues.apache.org/jira/browse/HADOOP-12180) | Move ResourceCalculatorPlugin from YARN to Common | Major | util | Chris Douglas | Chris Douglas | +| [HADOOP-12210](https://issues.apache.org/jira/browse/HADOOP-12210) | Collect network usage on the node | Major | . | Robert Grandl | Robert Grandl | +| [YARN-3069](https://issues.apache.org/jira/browse/YARN-3069) | Document missing properties in yarn-default.xml | Major | documentation | Ray Chiang | Ray Chiang | +| [YARN-3381](https://issues.apache.org/jira/browse/YARN-3381) | Fix typo InvalidStateTransitonException | Minor | api | Xiaoshuang LU | Brahma Reddy Battula | +| [HADOOP-12211](https://issues.apache.org/jira/browse/HADOOP-12211) | Collect disks usages on the node | Major | . | Robert Grandl | Robert Grandl | +| [HDFS-8722](https://issues.apache.org/jira/browse/HDFS-8722) | Optimize datanode writes for small writes and flushes | Critical | . | Kihwal Lee | Kihwal Lee | +| [HADOOP-12232](https://issues.apache.org/jira/browse/HADOOP-12232) | Upgrade Tomcat dependency to 6.0.44. | Major | build | Chris Nauroth | Chris Nauroth | +| [YARN-3170](https://issues.apache.org/jira/browse/YARN-3170) | YARN architecture document needs updating | Major | documentation | Allen Wittenauer | Brahma Reddy Battula | +| [MAPREDUCE-5762](https://issues.apache.org/jira/browse/MAPREDUCE-5762) | Port MAPREDUCE-3223 and MAPREDUCE-4695 (Remove MRv1 config from mapred-default.xml) to branch-2 | Minor | documentation | Akira Ajisaka | Akira Ajisaka | +| [YARN-3174](https://issues.apache.org/jira/browse/YARN-3174) | Consolidate the NodeManager and NodeManagerRestart documentation into one | Major | documentation | Allen Wittenauer | Masatake Iwasaki | +| [HDFS-7314](https://issues.apache.org/jira/browse/HDFS-7314) | When the DFSClient lease cannot be renewed, abort open-for-write files rather than the entire DFSClient | Major | . | Ming Ma | Ming Ma | +| [HADOOP-11893](https://issues.apache.org/jira/browse/HADOOP-11893) | Mark org.apache.hadoop.security.token.Token as @InterfaceAudience.Public | Major | security | Steve Loughran | Brahma Reddy Battula | +| [HADOOP-12081](https://issues.apache.org/jira/browse/HADOOP-12081) | Fix UserGroupInformation.java to support 64-bit zLinux | Major | security | Adam Roberts | Akira Ajisaka | +| [HADOOP-12214](https://issues.apache.org/jira/browse/HADOOP-12214) | Parse 'HadoopArchive' commandline using cli Options. | Minor | . | Vinayakumar B | Vinayakumar B | +| [YARN-2921](https://issues.apache.org/jira/browse/YARN-2921) | Fix MockRM/MockAM#waitForState sleep too long | Major | test | Karthik Kambatla | Tsuyoshi Ozawa | +| [HADOOP-12161](https://issues.apache.org/jira/browse/HADOOP-12161) | Add getStoragePolicy API to the FileSystem interface | Major | fs | Arpit Agarwal | Brahma Reddy Battula | +| [HADOOP-12189](https://issues.apache.org/jira/browse/HADOOP-12189) | Improve CallQueueManager#swapQueue to make queue elements drop nearly impossible. | Major | ipc, test | zhihai xu | zhihai xu | +| [HADOOP-12009](https://issues.apache.org/jira/browse/HADOOP-12009) | Clarify FileSystem.listStatus() sorting order & fix FileSystemContractBaseTest:testListStatus | Minor | documentation, fs, test | Jakob Homan | J.Andreina | +| [HADOOP-12259](https://issues.apache.org/jira/browse/HADOOP-12259) | Utility to Dynamic port allocation | Major | test, util | Brahma Reddy Battula | Brahma Reddy Battula | +| [HDFS-8735](https://issues.apache.org/jira/browse/HDFS-8735) | Inotify : All events classes should implement toString() API. | Major | hdfs-client | Surendra Singh Lilhore | Surendra Singh Lilhore | +| [HDFS-7858](https://issues.apache.org/jira/browse/HDFS-7858) | Improve HA Namenode Failover detection on the client | Major | hdfs-client | Arun Suresh | Arun Suresh | +| [HDFS-8180](https://issues.apache.org/jira/browse/HDFS-8180) | AbstractFileSystem Implementation for WebHdfs | Major | webhdfs | Santhosh G Nayak | Santhosh G Nayak | +| [HDFS-8811](https://issues.apache.org/jira/browse/HDFS-8811) | Move BlockStoragePolicy name's constants from HdfsServerConstants.java to HdfsConstants.java | Major | . | Vinayakumar B | Vinayakumar B | +| [HDFS-8822](https://issues.apache.org/jira/browse/HDFS-8822) | Add SSD storagepolicy tests in TestBlockStoragePolicy#testDefaultPolicies | Major | . | Vinayakumar B | Vinayakumar B | +| [YARN-3950](https://issues.apache.org/jira/browse/YARN-3950) | Add unique YARN\_SHELL\_ID environment variable to DistributedShell | Major | applications/distributed-shell | Robert Kanter | Robert Kanter | +| [YARN-2768](https://issues.apache.org/jira/browse/YARN-2768) | Avoid cloning Resource in FSAppAttempt#updateDemand | Minor | fairscheduler | Hong Zhiguo | Hong Zhiguo | +| [HDFS-8816](https://issues.apache.org/jira/browse/HDFS-8816) | Improve visualization for the Datanode tab in the NN UI | Major | . | Haohui Mai | Haohui Mai | +| [HDFS-8821](https://issues.apache.org/jira/browse/HDFS-8821) | Explain message "Operation category X is not supported in state standby" | Minor | . | Gautam Gopalakrishnan | Gautam Gopalakrishnan | +| [HADOOP-12271](https://issues.apache.org/jira/browse/HADOOP-12271) | Hadoop Jar Error Should Be More Explanatory | Minor | . | Jesse Anderson | Josh Elser | +| [HADOOP-12183](https://issues.apache.org/jira/browse/HADOOP-12183) | Annotate the HTrace span created by FsShell with the command-line arguments passed by the user | Minor | tracing | Masatake Iwasaki | Masatake Iwasaki | +| [YARN-3978](https://issues.apache.org/jira/browse/YARN-3978) | Configurably turn off the saving of container info in Generic AHS | Major | timelineserver, yarn | Eric Payne | Eric Payne | +| [YARN-3965](https://issues.apache.org/jira/browse/YARN-3965) | Add startup timestamp to nodemanager UI | Minor | nodemanager | Hong Zhiguo | Hong Zhiguo | +| [HADOOP-12280](https://issues.apache.org/jira/browse/HADOOP-12280) | Skip unit tests based on maven profile rather than NativeCodeLoader.isNativeCodeLoaded | Minor | test | Masatake Iwasaki | Masatake Iwasaki | +| [HDFS-8815](https://issues.apache.org/jira/browse/HDFS-8815) | DFS getStoragePolicy implementation using single RPC call | Major | hdfs-client | Arpit Agarwal | Surendra Singh Lilhore | +| [YARN-3961](https://issues.apache.org/jira/browse/YARN-3961) | Expose pending, running and reserved containers of a queue in REST api and yarn top | Major | capacityscheduler, fairscheduler, webapp | Anubhav Dhoot | Anubhav Dhoot | +| [YARN-4019](https://issues.apache.org/jira/browse/YARN-4019) | Add JvmPauseMonitor to ResourceManager and NodeManager | Major | nodemanager, resourcemanager | Robert Kanter | Robert Kanter | +| [MAPREDUCE-6443](https://issues.apache.org/jira/browse/MAPREDUCE-6443) | Add JvmPauseMonitor to Job History Server | Major | jobhistoryserver | Robert Kanter | Robert Kanter | +| [HDFS-8887](https://issues.apache.org/jira/browse/HDFS-8887) | Expose storage type and storage ID in BlockLocation | Major | . | Andrew Wang | Andrew Wang | +| [HADOOP-12318](https://issues.apache.org/jira/browse/HADOOP-12318) | Expose underlying LDAP exceptions in SaslPlainServer | Minor | security | Mike Yoder | Mike Yoder | +| [HADOOP-12295](https://issues.apache.org/jira/browse/HADOOP-12295) | Improve NetworkTopology#InnerNode#remove logic | Major | . | Yi Liu | Yi Liu | +| [HDFS-7649](https://issues.apache.org/jira/browse/HDFS-7649) | Multihoming docs should emphasize using hostnames in configurations | Major | documentation | Arpit Agarwal | Brahma Reddy Battula | +| [YARN-4055](https://issues.apache.org/jira/browse/YARN-4055) | Report node resource utilization in heartbeat | Major | nodemanager | Inigo Goiri | Inigo Goiri | +| [HDFS-8713](https://issues.apache.org/jira/browse/HDFS-8713) | Convert DatanodeDescriptor to use SLF4J logging | Trivial | . | Andrew Wang | Andrew Wang | +| [HDFS-8883](https://issues.apache.org/jira/browse/HDFS-8883) | NameNode Metrics : Add FSNameSystem lock Queue Length | Major | namenode | Anu Engineer | Anu Engineer | +| [HDFS-6407](https://issues.apache.org/jira/browse/HDFS-6407) | Add sorting and pagination in the datanode tab of the NN Web UI | Critical | namenode | Nathan Roberts | Haohui Mai | +| [HDFS-8880](https://issues.apache.org/jira/browse/HDFS-8880) | NameNode metrics logging | Major | namenode | Arpit Agarwal | Arpit Agarwal | +| [YARN-4057](https://issues.apache.org/jira/browse/YARN-4057) | If ContainersMonitor is not enabled, only print related log info one time | Minor | nodemanager | Jun Gong | Jun Gong | +| [HADOOP-12050](https://issues.apache.org/jira/browse/HADOOP-12050) | Enable MaxInactiveInterval for hadoop http auth token | Major | security | Benoy Antony | Huizhi Lu | +| [HDFS-8435](https://issues.apache.org/jira/browse/HDFS-8435) | Support CreateFlag in WebHdfs | Major | webhdfs | Vinoth Sathappan | Jakob Homan | +| [HDFS-8911](https://issues.apache.org/jira/browse/HDFS-8911) | NameNode Metric : Add Editlog counters as a JMX metric | Major | namenode | Anu Engineer | Anu Engineer | +| [HDFS-8917](https://issues.apache.org/jira/browse/HDFS-8917) | Cleanup BlockInfoUnderConstruction from comments and tests | Minor | namenode | Zhe Zhang | Zhe Zhang | +| [HDFS-8884](https://issues.apache.org/jira/browse/HDFS-8884) | Fail-fast check in BlockPlacementPolicyDefault#chooseTarget | Major | . | Yi Liu | Yi Liu | +| [HDFS-8828](https://issues.apache.org/jira/browse/HDFS-8828) | Utilize Snapshot diff report to build diff copy list in distcp | Major | distcp, snapshots | Yufei Gu | Yufei Gu | +| [HDFS-8924](https://issues.apache.org/jira/browse/HDFS-8924) | Add pluggable interface for reading replicas in DFSClient | Major | hdfs-client | Colin P. McCabe | Colin P. McCabe | +| [HDFS-8928](https://issues.apache.org/jira/browse/HDFS-8928) | Improvements for BlockUnderConstructionFeature: ReplicaUnderConstruction as a separate class and replicas as an array | Minor | namenode | Zhe Zhang | Jing Zhao | +| [HDFS-2390](https://issues.apache.org/jira/browse/HDFS-2390) | dfsadmin -setBalancerBandwidth doesnot validate -ve value | Minor | balancer & mover | Rajit Saha | Gautam Gopalakrishnan | +| [HDFS-8865](https://issues.apache.org/jira/browse/HDFS-8865) | Improve quota initialization performance | Major | . | Kihwal Lee | Kihwal Lee | +| [HDFS-8983](https://issues.apache.org/jira/browse/HDFS-8983) | NameNode support for protected directories | Major | namenode | Arpit Agarwal | Arpit Agarwal | +| [HDFS-8946](https://issues.apache.org/jira/browse/HDFS-8946) | Improve choosing datanode storage for block placement | Major | namenode | Yi Liu | Yi Liu | +| [HDFS-8965](https://issues.apache.org/jira/browse/HDFS-8965) | Harden edit log reading code against out of memory errors | Major | . | Colin P. McCabe | Colin P. McCabe | +| [HADOOP-12368](https://issues.apache.org/jira/browse/HADOOP-12368) | Mark ViewFileSystemBaseTest and ViewFsBaseTest as abstract | Trivial | . | Andrew Wang | Andrew Wang | +| [HADOOP-12367](https://issues.apache.org/jira/browse/HADOOP-12367) | Move TestFileUtil's test resources to resources folder | Minor | . | Andrew Wang | Andrew Wang | +| [HADOOP-12369](https://issues.apache.org/jira/browse/HADOOP-12369) | Point hadoop-project/pom.xml java.security.krb5.conf within target folder | Minor | . | Andrew Wang | Andrew Wang | +| [HDFS-328](https://issues.apache.org/jira/browse/HDFS-328) | Improve fs -setrep error message for invalid replication factors | Major | namenode | Tsz Wo Nicholas Sze | Daniel Templeton | +| [HADOOP-5323](https://issues.apache.org/jira/browse/HADOOP-5323) | Trash documentation should describe its directory structure and configurations | Minor | documentation | Suman Sehgal | Weiwei Yang | +| [HDFS-9021](https://issues.apache.org/jira/browse/HDFS-9021) | Use a yellow elephant rather than a blue one in diagram | Minor | . | Andrew Wang | Andrew Wang | +| [HADOOP-12358](https://issues.apache.org/jira/browse/HADOOP-12358) | Add -safely flag to rm to prompt when deleting many files | Major | fs | Xiaoyu Yao | Xiaoyu Yao | +| [YARN-4024](https://issues.apache.org/jira/browse/YARN-4024) | YARN RM should avoid unnecessary resolving IP when NMs doing heartbeat | Major | resourcemanager | Wangda Tan | Hong Zhiguo | +| [HADOOP-12384](https://issues.apache.org/jira/browse/HADOOP-12384) | Add "-direct" flag option for fs copy so that user can choose not to create ".\_COPYING\_" file | Major | fs | Chen He | J.Andreina | +| [HDFS-9019](https://issues.apache.org/jira/browse/HDFS-9019) | Adding informative message to sticky bit permission denied exception | Minor | security | Thejas M Nair | Xiaoyu Yao | +| [HDFS-8384](https://issues.apache.org/jira/browse/HDFS-8384) | Allow NN to startup if there are files having a lease but are not under construction | Minor | namenode | Tsz Wo Nicholas Sze | Jing Zhao | +| [HDFS-8929](https://issues.apache.org/jira/browse/HDFS-8929) | Add a metric to expose the timestamp of the last journal | Major | journal-node | Akira Ajisaka | Surendra Singh Lilhore | +| [HDFS-7116](https://issues.apache.org/jira/browse/HDFS-7116) | Add a command to get the balancer bandwidth | Major | balancer & mover | Akira Ajisaka | Rakesh R | +| [YARN-4086](https://issues.apache.org/jira/browse/YARN-4086) | Allow Aggregated Log readers to handle HAR files | Major | . | Robert Kanter | Robert Kanter | +| [HDFS-8974](https://issues.apache.org/jira/browse/HDFS-8974) | Convert docs in xdoc format to markdown | Minor | documentation | Masatake Iwasaki | Masatake Iwasaki | +| [YARN-4145](https://issues.apache.org/jira/browse/YARN-4145) | Make RMHATestBase abstract so its not run when running all tests under that namespace | Minor | . | Anubhav Dhoot | Anubhav Dhoot | +| [HADOOP-12324](https://issues.apache.org/jira/browse/HADOOP-12324) | Better exception reporting in SaslPlainServer | Minor | security | Mike Yoder | Mike Yoder | +| [YARN-2005](https://issues.apache.org/jira/browse/YARN-2005) | Blacklisting support for scheduling AMs | Major | resourcemanager | Jason Lowe | Anubhav Dhoot | +| [HDFS-8829](https://issues.apache.org/jira/browse/HDFS-8829) | Make SO\_RCVBUF and SO\_SNDBUF size configurable for DataTransferProtocol sockets and allow configuring auto-tuning | Major | datanode | He Tianyi | He Tianyi | +| [HDFS-9065](https://issues.apache.org/jira/browse/HDFS-9065) | Include commas on # of files, blocks, total filesystem objects in NN Web UI | Minor | namenode | Daniel Templeton | Daniel Templeton | +| [HADOOP-12413](https://issues.apache.org/jira/browse/HADOOP-12413) | AccessControlList should avoid calling getGroupNames in isUserInList with empty groups. | Major | security | zhihai xu | zhihai xu | +| [HDFS-8953](https://issues.apache.org/jira/browse/HDFS-8953) | DataNode Metrics logging | Major | . | Kanaka Kumar Avvaru | Kanaka Kumar Avvaru | +| [YARN-4158](https://issues.apache.org/jira/browse/YARN-4158) | Remove duplicate close for LogWriter in AppLogAggregatorImpl#uploadLogsForContainers | Minor | nodemanager | zhihai xu | zhihai xu | +| [YARN-4149](https://issues.apache.org/jira/browse/YARN-4149) | yarn logs -am should provide an option to fetch all the log files | Major | client, nodemanager | Varun Vasudev | Varun Vasudev | +| [HDFS-9082](https://issues.apache.org/jira/browse/HDFS-9082) | Change the log level in WebHdfsFileSystem.initialize() from INFO to DEBUG | Minor | webhdfs | Santhosh G Nayak | Santhosh G Nayak | +| [YARN-4135](https://issues.apache.org/jira/browse/YARN-4135) | Improve the assertion message in MockRM while failing after waiting for the state. | Trivial | . | nijel | nijel | +| [MAPREDUCE-6478](https://issues.apache.org/jira/browse/MAPREDUCE-6478) | Add an option to skip cleanupJob stage or ignore cleanup failure during commitJob(). | Major | . | Junping Du | Junping Du | +| [HADOOP-12404](https://issues.apache.org/jira/browse/HADOOP-12404) | Disable caching for JarURLConnection to avoid sharing JarFile with other users when loading resource from URL in Configuration class. | Minor | conf | zhihai xu | zhihai xu | +| [HADOOP-12428](https://issues.apache.org/jira/browse/HADOOP-12428) | Fix inconsistency between log-level guards and statements | Minor | . | Jackie Chang | Jagadesh Kiran N | +| [YARN-4095](https://issues.apache.org/jira/browse/YARN-4095) | Avoid sharing AllocatorPerContext object in LocalDirAllocator between ShuffleHandler and LocalDirsHandlerService. | Major | nodemanager | zhihai xu | zhihai xu | +| [HDFS-5795](https://issues.apache.org/jira/browse/HDFS-5795) | RemoteBlockReader2#checkSuccess() shoud print error status | Trivial | . | Brandon Li | Xiao Chen | +| [HDFS-9112](https://issues.apache.org/jira/browse/HDFS-9112) | Improve error message for Haadmin when multiple name service IDs are configured | Major | tools | Anu Engineer | Anu Engineer | +| [HDFS-9132](https://issues.apache.org/jira/browse/HDFS-9132) | Pass genstamp to ReplicaAccessorBuilder | Major | hdfs-client | Colin P. McCabe | Colin P. McCabe | +| [HDFS-8873](https://issues.apache.org/jira/browse/HDFS-8873) | Allow the directoryScanner to be rate-limited | Major | datanode | Nathan Roberts | Daniel Templeton | +| [HADOOP-12442](https://issues.apache.org/jira/browse/HADOOP-12442) | Display help if the command option to "hdfs dfs " is not valid | Minor | . | nijel | nijel | +| [HADOOP-11984](https://issues.apache.org/jira/browse/HADOOP-11984) | Enable parallel JUnit tests in pre-commit. | Major | build, scripts, test | Chris Nauroth | Chris Nauroth | +| [MAPREDUCE-6471](https://issues.apache.org/jira/browse/MAPREDUCE-6471) | Document distcp incremental copy | Major | distcp | Arpit Agarwal | Neelesh Srinivas Salian | +| [HDFS-9148](https://issues.apache.org/jira/browse/HDFS-9148) | Incorrect assert message in TestWriteToReplica#testWriteToTemporary | Trivial | test | Tony Wu | Tony Wu | +| [HDFS-8859](https://issues.apache.org/jira/browse/HDFS-8859) | Improve DataNode ReplicaMap memory footprint to save about 45% | Major | datanode | Yi Liu | Yi Liu | +| [HDFS-8696](https://issues.apache.org/jira/browse/HDFS-8696) | Make the lower and higher watermark in the DN Netty server configurable | Major | webhdfs | Xiaobing Zhou | Xiaobing Zhou | +| [YARN-3727](https://issues.apache.org/jira/browse/YARN-3727) | For better error recovery, check if the directory exists before using it for localization. | Major | nodemanager | zhihai xu | zhihai xu | +| [HDFS-9175](https://issues.apache.org/jira/browse/HDFS-9175) | Change scope of 'AccessTokenProvider.getAccessToken()' and 'CredentialBasedAccessTokenProvider.getCredential()' abstract methods to public | Major | webhdfs | Santhosh G Nayak | Santhosh G Nayak | +| [HADOOP-12458](https://issues.apache.org/jira/browse/HADOOP-12458) | Retries is typoed to spell Retires in parts of hadoop-yarn and hadoop-common | Minor | documentation | Neelesh Srinivas Salian | Neelesh Srinivas Salian | +| [HDFS-9151](https://issues.apache.org/jira/browse/HDFS-9151) | Mover should print the exit status/reason on console like balancer tool. | Minor | balancer & mover | Archana T | Surendra Singh Lilhore | +| [HADOOP-12350](https://issues.apache.org/jira/browse/HADOOP-12350) | WASB Logging: Improve WASB Logging around deletes, reads and writes | Major | tools | Dushyanth | Dushyanth | +| [HADOOP-12284](https://issues.apache.org/jira/browse/HADOOP-12284) | UserGroupInformation doAs can throw misleading exception | Trivial | security | Aaron Dossett | Aaron Dossett | +| [YARN-4228](https://issues.apache.org/jira/browse/YARN-4228) | FileSystemRMStateStore use IOUtils#close instead of fs#close | Minor | . | Bibin A Chundatt | Bibin A Chundatt | +| [YARN-3943](https://issues.apache.org/jira/browse/YARN-3943) | Use separate threshold configurations for disk-full detection and disk-not-full detection. | Critical | nodemanager | zhihai xu | zhihai xu | +| [MAPREDUCE-6479](https://issues.apache.org/jira/browse/MAPREDUCE-6479) | Add missing mapred job command options in mapreduce document | Major | documentation | nijel | nijel | +| [HADOOP-11104](https://issues.apache.org/jira/browse/HADOOP-11104) | org.apache.hadoop.metrics2.lib.MetricsRegistry needs numerical parameter checking | Minor | . | Ray Chiang | Ray Chiang | +| [HDFS-9181](https://issues.apache.org/jira/browse/HDFS-9181) | Better handling of exceptions thrown during upgrade shutdown | Minor | . | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HDFS-9110](https://issues.apache.org/jira/browse/HDFS-9110) | Use Files.walkFileTree in NNUpgradeUtil#doPreUpgrade for better efficiency | Minor | . | Charlie Helin | Charlie Helin | +| [HDFS-9221](https://issues.apache.org/jira/browse/HDFS-9221) | HdfsServerConstants#ReplicaState#getState should avoid calling values() since it creates a temporary array | Major | performance | Staffan Friberg | Staffan Friberg | +| [HDFS-8988](https://issues.apache.org/jira/browse/HDFS-8988) | Use LightWeightHashSet instead of LightWeightLinkedSet in BlockManager#excessReplicateMap | Major | . | Yi Liu | Yi Liu | +| [HDFS-9139](https://issues.apache.org/jira/browse/HDFS-9139) | Enable parallel JUnit tests for HDFS Pre-commit | Major | test | Vinayakumar B | Vinayakumar B | +| [HDFS-9145](https://issues.apache.org/jira/browse/HDFS-9145) | Tracking methods that hold FSNamesytemLock for too long | Major | namenode | Jing Zhao | Mingliang Liu | +| [HADOOP-10775](https://issues.apache.org/jira/browse/HADOOP-10775) | Shell operations to fail with meaningful errors on windows if winutils.exe not found | Minor | util | Steve Loughran | Steve Loughran | +| [YARN-4253](https://issues.apache.org/jira/browse/YARN-4253) | Standardize on using PrivilegedOperationExecutor for all invocations of container-executor in LinuxContainerExecutor | Major | . | Sidharta Seethana | Sidharta Seethana | +| [YARN-4252](https://issues.apache.org/jira/browse/YARN-4252) | Log container-executor invocation details when exit code is non-zero | Minor | nodemanager | Sidharta Seethana | Sidharta Seethana | +| [HDFS-9238](https://issues.apache.org/jira/browse/HDFS-9238) | Update TestFileCreation#testLeaseExpireHardLimit() to avoid using DataNodeTestUtils#getFile() | Trivial | test | Tony Wu | Tony Wu | +| [HDFS-9188](https://issues.apache.org/jira/browse/HDFS-9188) | Make block corruption related tests FsDataset-agnostic. | Major | test | Lei (Eddy) Xu | Lei (Eddy) Xu | +| [HDFS-9205](https://issues.apache.org/jira/browse/HDFS-9205) | Do not schedule corrupt blocks for replication | Minor | namenode | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze | +| [HADOOP-12481](https://issues.apache.org/jira/browse/HADOOP-12481) | JWTRedirectAuthenticationHandler doesn't Retain Original Query String | Major | security | Larry McCay | Larry McCay | +| [HDFS-9257](https://issues.apache.org/jira/browse/HDFS-9257) | improve error message for "Absolute path required" in INode.java to contain the rejected path | Trivial | namenode | Marcell Szabo | Marcell Szabo | +| [HDFS-9253](https://issues.apache.org/jira/browse/HDFS-9253) | Refactor tests of libhdfs into a directory | Major | . | Haohui Mai | Haohui Mai | +| [HADOOP-12450](https://issues.apache.org/jira/browse/HADOOP-12450) | UserGroupInformation should not log at WARN level if no groups are found | Minor | security | Elliott Clark | Elliott Clark | +| [HADOOP-12460](https://issues.apache.org/jira/browse/HADOOP-12460) | Add overwrite option for 'get' shell command | Major | . | Keegan Witt | Jagadesh Kiran N | +| [HDFS-9250](https://issues.apache.org/jira/browse/HDFS-9250) | Add Precondition check to LocatedBlock#addCachedLoc | Major | namenode | Xiao Chen | Xiao Chen | +| [HDFS-9251](https://issues.apache.org/jira/browse/HDFS-9251) | Refactor TestWriteToReplica and TestFsDatasetImpl to avoid explicitly creating Files in tests code. | Major | test | Lei (Eddy) Xu | Lei (Eddy) Xu | +| [MAPREDUCE-6489](https://issues.apache.org/jira/browse/MAPREDUCE-6489) | Fail fast rogue tasks that write too much to local disk | Major | task | Maysam Yabandeh | Maysam Yabandeh | +| [HDFS-8647](https://issues.apache.org/jira/browse/HDFS-8647) | Abstract BlockManager's rack policy into BlockPlacementPolicy | Major | . | Ming Ma | Brahma Reddy Battula | +| [HDFS-7087](https://issues.apache.org/jira/browse/HDFS-7087) | Ability to list /.reserved | Major | . | Andrew Wang | Xiao Chen | +| [HDFS-9280](https://issues.apache.org/jira/browse/HDFS-9280) | Document NFS gateway export point parameter | Trivial | documentation | Zhe Zhang | Xiao Chen | +| [HADOOP-12334](https://issues.apache.org/jira/browse/HADOOP-12334) | Change Mode Of Copy Operation of HBase WAL Archiving to bypass Azure Storage Throttling after retries | Major | tools | Gaurav Kanade | Gaurav Kanade | +| [HADOOP-7266](https://issues.apache.org/jira/browse/HADOOP-7266) | Deprecate metrics v1 | Blocker | metrics | Luke Lu | Akira Ajisaka | +| [YARN-2913](https://issues.apache.org/jira/browse/YARN-2913) | Fair scheduler should have ability to set MaxResourceDefault for each queue | Major | . | Siqi Li | Siqi Li | +| [HDFS-9264](https://issues.apache.org/jira/browse/HDFS-9264) | Minor cleanup of operations on FsVolumeList#volumes | Minor | . | Walter Su | Walter Su | +| [HDFS-8808](https://issues.apache.org/jira/browse/HDFS-8808) | dfs.image.transfer.bandwidthPerSec should not apply to -bootstrapStandby | Major | . | Gautam Gopalakrishnan | Zhe Zhang | +| [HDFS-9297](https://issues.apache.org/jira/browse/HDFS-9297) | Update TestBlockMissingException to use corruptBlockOnDataNodesByDeletingBlockFile() | Trivial | test | Tony Wu | Tony Wu | +| [HDFS-4015](https://issues.apache.org/jira/browse/HDFS-4015) | Safemode should count and report orphaned blocks | Major | namenode | Todd Lipcon | Anu Engineer | +| [YARN-3528](https://issues.apache.org/jira/browse/YARN-3528) | Tests with 12345 as hard-coded port break jenkins | Blocker | . | Steve Loughran | Brahma Reddy Battula | +| [YARN-4285](https://issues.apache.org/jira/browse/YARN-4285) | Display resource usage as percentage of queue and cluster in the RM UI | Major | resourcemanager | Varun Vasudev | Varun Vasudev | +| [HDFS-7284](https://issues.apache.org/jira/browse/HDFS-7284) | Add more debug info to BlockInfoUnderConstruction#setGenerationStampAndVerifyReplicas | Major | namenode | Hu Liu, | Wei-Chiu Chuang | +| [HADOOP-12472](https://issues.apache.org/jira/browse/HADOOP-12472) | Make GenericTestUtils.assertExceptionContains robust | Minor | test | Steve Loughran | Steve Loughran | +| [HDFS-9291](https://issues.apache.org/jira/browse/HDFS-9291) | Fix TestInterDatanodeProtocol to be FsDataset-agnostic. | Minor | test | Lei (Eddy) Xu | Lei (Eddy) Xu | +| [HDFS-8945](https://issues.apache.org/jira/browse/HDFS-8945) | Update the description about replica placement in HDFS Architecture documentation | Minor | documentation | Masatake Iwasaki | Masatake Iwasaki | +| [HDFS-9292](https://issues.apache.org/jira/browse/HDFS-9292) | Make TestFileConcorruption independent to underlying FsDataset Implementation. | Minor | . | Lei (Eddy) Xu | Lei (Eddy) Xu | +| [HDFS-9259](https://issues.apache.org/jira/browse/HDFS-9259) | Make SO\_SNDBUF size configurable at DFSClient side for hdfs write scenario | Major | . | Ming Ma | Mingliang Liu | +| [HDFS-9299](https://issues.apache.org/jira/browse/HDFS-9299) | Give ReplicationMonitor a readable thread name | Trivial | namenode | Staffan Friberg | Staffan Friberg | +| [HDFS-9307](https://issues.apache.org/jira/browse/HDFS-9307) | fuseConnect should be private to fuse\_connect.c | Trivial | fuse-dfs | Colin P. McCabe | Mingliang Liu | +| [HADOOP-12520](https://issues.apache.org/jira/browse/HADOOP-12520) | Use XInclude in hadoop-azure test configuration to isolate Azure Storage account keys for service integration tests. | Major | fs/azure, test | Chris Nauroth | Chris Nauroth | +| [HDFS-9311](https://issues.apache.org/jira/browse/HDFS-9311) | Support optional offload of NameNode HA service health checks to a separate RPC server. | Major | ha, namenode | Chris Nauroth | Chris Nauroth | +| [HDFS-9255](https://issues.apache.org/jira/browse/HDFS-9255) | Consolidate block recovery related implementation into a single class | Minor | datanode | Walter Su | Walter Su | +| [YARN-2573](https://issues.apache.org/jira/browse/YARN-2573) | Integrate ReservationSystem with the RM failover mechanism | Major | capacityscheduler, fairscheduler, resourcemanager | Subru Krishnan | Subru Krishnan | +| [HDFS-6200](https://issues.apache.org/jira/browse/HDFS-6200) | Create a separate jar for hdfs-client | Major | build | Haohui Mai | Haohui Mai | +| [HDFS-8545](https://issues.apache.org/jira/browse/HDFS-8545) | Refactor FS#getUsed() to use ContentSummary and add an API to fetch the total file length from a specific path | Minor | . | J.Andreina | J.Andreina | +| [HDFS-9229](https://issues.apache.org/jira/browse/HDFS-9229) | Expose size of NameNode directory as a metric | Minor | namenode | Zhe Zhang | Surendra Singh Lilhore | +| [YARN-4310](https://issues.apache.org/jira/browse/YARN-4310) | FairScheduler: Log skipping reservation messages at DEBUG level | Minor | fairscheduler | Arun Suresh | Arun Suresh | +| [HDFS-9312](https://issues.apache.org/jira/browse/HDFS-9312) | Fix TestReplication to be FsDataset-agnostic. | Minor | . | Lei (Eddy) Xu | Lei (Eddy) Xu | +| [HDFS-9308](https://issues.apache.org/jira/browse/HDFS-9308) | Add truncateMeta() and deleteMeta() to MiniDFSCluster | Minor | test | Tony Wu | Tony Wu | +| [HDFS-9331](https://issues.apache.org/jira/browse/HDFS-9331) | Modify TestNameNodeMXBean#testNameNodeMXBeanInfo() to account for filesystem entirely allocated for DFS use | Trivial | test | Tony Wu | Tony Wu | +| [HDFS-9363](https://issues.apache.org/jira/browse/HDFS-9363) | Add fetchReplica() to FsDatasetTestUtils to return FsDataset-agnostic Replica. | Minor | test | Tony Wu | Tony Wu | +| [HDFS-9282](https://issues.apache.org/jira/browse/HDFS-9282) | Make data directory count and storage raw capacity related tests FsDataset-agnostic | Minor | test | Tony Wu | Tony Wu | +| [HADOOP-12344](https://issues.apache.org/jira/browse/HADOOP-12344) | Improve validateSocketPathSecurity0 error message | Trivial | net | Casey Brotherton | Casey Brotherton | +| [HDFS-9398](https://issues.apache.org/jira/browse/HDFS-9398) | Make ByteArraryManager log message in one-line format | Minor | hdfs-client | Mingliang Liu | Mingliang Liu | +| [HDFS-9234](https://issues.apache.org/jira/browse/HDFS-9234) | WebHdfs : getContentSummary() should give quota for storage types | Major | webhdfs | Surendra Singh Lilhore | Surendra Singh Lilhore | +| [HDFS-9369](https://issues.apache.org/jira/browse/HDFS-9369) | Use ctest to run tests for hadoop-hdfs-native-client | Minor | . | Haohui Mai | Haohui Mai | +| [HADOOP-12562](https://issues.apache.org/jira/browse/HADOOP-12562) | Make hadoop dockerfile usable by Yetus | Major | build | Allen Wittenauer | Allen Wittenauer | +| [YARN-4287](https://issues.apache.org/jira/browse/YARN-4287) | Capacity Scheduler: Rack Locality improvement | Major | capacityscheduler | Nathan Roberts | Nathan Roberts | +| [MAPREDUCE-5485](https://issues.apache.org/jira/browse/MAPREDUCE-5485) | Allow repeating job commit by extending OutputCommitter API | Critical | . | Nemon Lou | Junping Du | +| [MAPREDUCE-6499](https://issues.apache.org/jira/browse/MAPREDUCE-6499) | Add elapsed time for retired job in JobHistoryServer WebUI | Major | webapps | Yiqun Lin | Yiqun Lin | +| [HDFS-9252](https://issues.apache.org/jira/browse/HDFS-9252) | Change TestFileTruncate to use FsDatasetTestUtils to get block file size and genstamp. | Major | . | Lei (Eddy) Xu | Lei (Eddy) Xu | +| [HADOOP-12568](https://issues.apache.org/jira/browse/HADOOP-12568) | Update core-default.xml to describe posixGroups support | Minor | documentation | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [YARN-4279](https://issues.apache.org/jira/browse/YARN-4279) | Mark ApplicationId and ApplicationAttemptId static methods as @Public, @Unstable | Minor | client | Steve Loughran | Steve Loughran | +| [HADOOP-12575](https://issues.apache.org/jira/browse/HADOOP-12575) | Add build instruction for docker toolbox instead of boot2docker | Trivial | documentation | Kai Sasaki | Kai Sasaki | +| [HDFS-8056](https://issues.apache.org/jira/browse/HDFS-8056) | Decommissioned dead nodes should continue to be counted as dead after NN restart | Major | . | Ming Ma | Ming Ma | +| [HADOOP-11901](https://issues.apache.org/jira/browse/HADOOP-11901) | BytesWritable fails to support 2G chunks due to integer overflow | Major | . | Reynold Xin | Reynold Xin | +| [HDFS-9439](https://issues.apache.org/jira/browse/HDFS-9439) | Include status of closeAck into exception message in DataNode#run | Trivial | . | Xiao Chen | Xiao Chen | +| [HDFS-9402](https://issues.apache.org/jira/browse/HDFS-9402) | Switch DataNode.LOG to use slf4j | Minor | . | Walter Su | Walter Su | +| [HDFS-3302](https://issues.apache.org/jira/browse/HDFS-3302) | Review and improve HDFS trash documentation | Major | documentation | Harsh J | | +| [HADOOP-10035](https://issues.apache.org/jira/browse/HADOOP-10035) | Cleanup TestFilterFileSystem | Major | . | Suresh Srinivas | Suresh Srinivas | +| [HADOOP-10555](https://issues.apache.org/jira/browse/HADOOP-10555) | Add offset support to MurmurHash | Trivial | . | Sergey Shelukhin | Sergey Shelukhin | +| [HADOOP-10068](https://issues.apache.org/jira/browse/HADOOP-10068) | Improve log4j regex in testFindContainingJar | Trivial | . | Robert Rati | Robert Rati | +| [HDFS-9024](https://issues.apache.org/jira/browse/HDFS-9024) | Deprecate the TotalFiles metric | Major | . | Akira Ajisaka | Akira Ajisaka | +| [HDFS-7988](https://issues.apache.org/jira/browse/HDFS-7988) | Replace usage of ExactSizeInputStream with LimitInputStream. | Minor | . | Chris Nauroth | Walter Su | +| [HDFS-9314](https://issues.apache.org/jira/browse/HDFS-9314) | Improve BlockPlacementPolicyDefault's picking of excess replicas | Major | . | Ming Ma | Xiao Chen | +| [MAPREDUCE-5870](https://issues.apache.org/jira/browse/MAPREDUCE-5870) | Support for passing Job priority through Application Submission Context in Mapreduce Side | Major | client | Sunil G | Sunil G | +| [HDFS-9434](https://issues.apache.org/jira/browse/HDFS-9434) | Recommission a datanode with 500k blocks may pause NN for 30 seconds | Major | namenode | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze | +| [YARN-4132](https://issues.apache.org/jira/browse/YARN-4132) | Separate configs for nodemanager to resourcemanager connection timeout and retries | Major | nodemanager | Chang Li | Chang Li | +| [HDFS-8512](https://issues.apache.org/jira/browse/HDFS-8512) | WebHDFS : GETFILESTATUS should return LocatedBlock with storage type info | Major | webhdfs | Sumana Sathish | Xiaoyu Yao | +| [HADOOP-12600](https://issues.apache.org/jira/browse/HADOOP-12600) | FileContext and AbstractFileSystem should be annotated as a Stable interface. | Blocker | fs | Chris Nauroth | Chris Nauroth | +| [HDFS-9269](https://issues.apache.org/jira/browse/HDFS-9269) | Update the documentation and wrapper for fuse-dfs | Minor | fuse-dfs | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HDFS-9485](https://issues.apache.org/jira/browse/HDFS-9485) | Make BlockManager#removeFromExcessReplicateMap accept BlockInfo instead of Block | Minor | namenode | Mingliang Liu | Mingliang Liu | +| [HDFS-9490](https://issues.apache.org/jira/browse/HDFS-9490) | MiniDFSCluster should change block generation stamp via FsDatasetTestUtils | Minor | test | Tony Wu | Tony Wu | +| [HDFS-8831](https://issues.apache.org/jira/browse/HDFS-8831) | Trash Support for deletion in HDFS encryption zone | Major | encryption | Xiaoyu Yao | Xiaoyu Yao | +| [HDFS-9474](https://issues.apache.org/jira/browse/HDFS-9474) | TestPipelinesFailover should not fail when printing debug message | Major | . | Yongjun Zhang | John Zhuge | +| [YARN-3456](https://issues.apache.org/jira/browse/YARN-3456) | Improve handling of incomplete TimelineEntities | Minor | timelineserver | Steve Loughran | Varun Saxena | +| [HDFS-9527](https://issues.apache.org/jira/browse/HDFS-9527) | The return type of FSNamesystem.getBlockCollection should be changed to INodeFile | Minor | namenode | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze | +| [HDFS-9472](https://issues.apache.org/jira/browse/HDFS-9472) | concat() API does not give proper exception messages on ./reserved relative path | Major | namenode | Rakesh R | Rakesh R | +| [HDFS-9532](https://issues.apache.org/jira/browse/HDFS-9532) | Detailed exception info is lost in reportTo method of ErrorReportAction and ReportBadBlockAction | Trivial | datanode | Yongjun Zhang | Yongjun Zhang | +| [HDFS-9528](https://issues.apache.org/jira/browse/HDFS-9528) | Cleanup namenode audit/log/exception messages | Minor | namenode | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze | +| [HDFS-8860](https://issues.apache.org/jira/browse/HDFS-8860) | Remove unused Replica copyOnWrite code | Major | . | Lei (Eddy) Xu | Lei (Eddy) Xu | +| [MAPREDUCE-6436](https://issues.apache.org/jira/browse/MAPREDUCE-6436) | JobHistory cache issue | Blocker | . | Ryu Kobayashi | Kai Sasaki | +| [HADOOP-12639](https://issues.apache.org/jira/browse/HADOOP-12639) | Imrpove JavaDoc for getTrimmedStrings | Trivial | util | BELUGA BEHR | BELUGA BEHR | +| [HDFS-9557](https://issues.apache.org/jira/browse/HDFS-9557) | Reduce object allocation in PB conversion | Major | hdfs-client | Daryn Sharp | Daryn Sharp | +| [YARN-4207](https://issues.apache.org/jira/browse/YARN-4207) | Add a non-judgemental YARN app completion status | Major | . | Sergey Shelukhin | Rich Haase | +| [HDFS-9198](https://issues.apache.org/jira/browse/HDFS-9198) | Coalesce IBR processing in the NN | Major | namenode | Daryn Sharp | Daryn Sharp | +| [HDFS-9552](https://issues.apache.org/jira/browse/HDFS-9552) | Document types of permission checks performed for HDFS operations. | Major | documentation | Chris Nauroth | Chris Nauroth | +| [HADOOP-12570](https://issues.apache.org/jira/browse/HADOOP-12570) | HDFS Secure Mode Documentation updates | Major | documentation | Arpit Agarwal | Arpit Agarwal | +| [YARN-4480](https://issues.apache.org/jira/browse/YARN-4480) | Clean up some inappropriate imports | Major | . | Kai Zheng | Kai Zheng | +| [YARN-4290](https://issues.apache.org/jira/browse/YARN-4290) | Add -showDetails option to YARN Nodes CLI to print all nodes reports information | Major | client | Wangda Tan | Sunil G | +| [YARN-4400](https://issues.apache.org/jira/browse/YARN-4400) | AsyncDispatcher.waitForDrained should be final | Trivial | yarn | Daniel Templeton | Daniel Templeton | +| [MAPREDUCE-6584](https://issues.apache.org/jira/browse/MAPREDUCE-6584) | Remove trailing whitespaces from mapred-default.xml | Major | documentation | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-12686](https://issues.apache.org/jira/browse/HADOOP-12686) | Update FileSystemShell documentation to mention the meaning of each columns of fs -du | Minor | documentation, fs | Daisuke Kobayashi | Daisuke Kobayashi | +| [YARN-4544](https://issues.apache.org/jira/browse/YARN-4544) | All the log messages about rolling monitoring interval are shown with WARN level | Minor | log-aggregation, nodemanager | Takashi Ohnishi | Takashi Ohnishi | +| [YARN-4438](https://issues.apache.org/jira/browse/YARN-4438) | Implement RM leader election with curator | Major | . | Jian He | Jian He | +| [HDFS-9630](https://issues.apache.org/jira/browse/HDFS-9630) | DistCp minor refactoring and clean up | Minor | distcp | Kai Zheng | Kai Zheng | +| [YARN-4582](https://issues.apache.org/jira/browse/YARN-4582) | Label-related invalid resource request exception should be able to properly handled by application | Major | scheduler | Bibin A Chundatt | Bibin A Chundatt | +| [HDFS-9569](https://issues.apache.org/jira/browse/HDFS-9569) | Log the name of the fsimage being loaded for better supportability | Trivial | namenode | Yongjun Zhang | Yongjun Zhang | +| [MAPREDUCE-6473](https://issues.apache.org/jira/browse/MAPREDUCE-6473) | Job submission can take a long time during Cluster initialization | Major | performance | Kuhu Shukla | Kuhu Shukla | +| [HDFS-9415](https://issues.apache.org/jira/browse/HDFS-9415) | Document dfs.cluster.administrators and dfs.permissions.superusergroup | Major | documentation | Arpit Agarwal | Xiaobing Zhou | +| [HDFS-6054](https://issues.apache.org/jira/browse/HDFS-6054) | MiniQJMHACluster should not use static port to avoid binding failure in unit test | Major | test | Brandon Li | Yongjun Zhang | +| [YARN-4492](https://issues.apache.org/jira/browse/YARN-4492) | Add documentation for preemption supported in Capacity scheduler | Minor | capacity scheduler | Naganarasimha G R | Naganarasimha G R | +| [YARN-4371](https://issues.apache.org/jira/browse/YARN-4371) | "yarn application -kill" should take multiple application ids | Major | . | Tsuyoshi Ozawa | Sunil G | +| [HDFS-9653](https://issues.apache.org/jira/browse/HDFS-9653) | Expose the number of blocks pending deletion through dfsadmin report command | Major | hdfs-client, tools | Weiwei Yang | Weiwei Yang | +| [HADOOP-12731](https://issues.apache.org/jira/browse/HADOOP-12731) | Remove useless boxing/unboxing code | Minor | performance | Kousuke Saruta | Kousuke Saruta | +| [HDFS-9654](https://issues.apache.org/jira/browse/HDFS-9654) | Code refactoring for HDFS-8578 | Minor | datanode | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze | +| [HDFS-9706](https://issues.apache.org/jira/browse/HDFS-9706) | Log more details in debug logs in BlockReceiver's constructor | Minor | . | Xiao Chen | Xiao Chen | +| [HDFS-9638](https://issues.apache.org/jira/browse/HDFS-9638) | Improve DistCp Help and documentation | Minor | distcp | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HDFS-9566](https://issues.apache.org/jira/browse/HDFS-9566) | Remove expensive 'BlocksMap#getStorages(Block b, final DatanodeStorage.State state)' method | Major | namenode | Daryn Sharp | Daryn Sharp | +| [HDFS-9721](https://issues.apache.org/jira/browse/HDFS-9721) | Allow Delimited PB OIV tool to run upon fsimage that contains INodeReference | Major | . | Xiao Chen | Xiao Chen | +| [HDFS-9669](https://issues.apache.org/jira/browse/HDFS-9669) | TcpPeerServer should respect ipc.server.listen.queue.size | Major | . | Elliott Clark | Elliott Clark | +| [HDFS-9715](https://issues.apache.org/jira/browse/HDFS-9715) | Check storage ID uniqueness on datanode startup | Major | datanode | Lei (Eddy) Xu | Lei (Eddy) Xu | +| [YARN-4662](https://issues.apache.org/jira/browse/YARN-4662) | Document some newly added metrics | Major | . | Jian He | Jian He | +| [HDFS-9629](https://issues.apache.org/jira/browse/HDFS-9629) | Update the footer of Web UI to show year 2016 | Major | . | Xiao Chen | Xiao Chen | +| [MAPREDUCE-6566](https://issues.apache.org/jira/browse/MAPREDUCE-6566) | Add retry support to mapreduce CLI tool | Major | . | Varun Vasudev | Varun Vasudev | +| [HDFS-9726](https://issues.apache.org/jira/browse/HDFS-9726) | Refactor IBR code to a new class | Minor | datanode | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze | +| [HADOOP-12772](https://issues.apache.org/jira/browse/HADOOP-12772) | NetworkTopologyWithNodeGroup.getNodeGroup() can loop infinitely for invalid 'loc' values | Minor | . | Kuhu Shukla | Kuhu Shukla | +| [HADOOP-12758](https://issues.apache.org/jira/browse/HADOOP-12758) | Extend CSRF Filter with UserAgent Checks | Major | security | Larry McCay | Larry McCay | +| [HDFS-9686](https://issues.apache.org/jira/browse/HDFS-9686) | Remove useless boxing/unboxing code | Minor | performance | Kousuke Saruta | Kousuke Saruta | +| [MAPREDUCE-6626](https://issues.apache.org/jira/browse/MAPREDUCE-6626) | Reuse ObjectMapper instance in MapReduce | Minor | performance | Yiqun Lin | Yiqun Lin | +| [HADOOP-12788](https://issues.apache.org/jira/browse/HADOOP-12788) | OpensslAesCtrCryptoCodec should log which random number generator is used. | Minor | . | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HADOOP-12764](https://issues.apache.org/jira/browse/HADOOP-12764) | Increase default value of KMS maxHttpHeaderSize and make it configurable | Minor | . | Zhe Zhang | Zhe Zhang | +| [HADOOP-12776](https://issues.apache.org/jira/browse/HADOOP-12776) | Remove getaclstatus call for non-acl commands in getfacl. | Major | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [HDFS-9768](https://issues.apache.org/jira/browse/HDFS-9768) | Reuse objectMapper instance in HDFS to improve the performance | Major | performance | Yiqun Lin | Yiqun Lin | +| [HDFS-9644](https://issues.apache.org/jira/browse/HDFS-9644) | Update encryption documentation to reflect nested EZs | Major | documentation, encryption | Zhe Zhang | Zhe Zhang | +| [HDFS-9700](https://issues.apache.org/jira/browse/HDFS-9700) | DFSClient and DFSOutputStream should set TCP\_NODELAY on sockets for DataTransferProtocol | Major | hdfs-client | Gary Helmling | Gary Helmling | +| [HDFS-9797](https://issues.apache.org/jira/browse/HDFS-9797) | Log Standby exceptions thrown by RequestHedgingProxyProvider at DEBUG Level | Minor | hdfs-client | Inigo Goiri | Inigo Goiri | +| [YARN-4682](https://issues.apache.org/jira/browse/YARN-4682) | AMRM client to log when AMRM token updated | Major | client | Steve Loughran | Prabhu Joseph | +| [HADOOP-12805](https://issues.apache.org/jira/browse/HADOOP-12805) | Annotate CanUnbuffer with @InterfaceAudience.Public | Major | . | Ted Yu | Ted Yu | +| [YARN-4690](https://issues.apache.org/jira/browse/YARN-4690) | Skip object allocation in FSAppAttempt#getResourceUsage when possible | Major | . | Ming Ma | Ming Ma | +| [HADOOP-10865](https://issues.apache.org/jira/browse/HADOOP-10865) | Add a Crc32 chunked verification benchmark for both directly and non-directly buffer cases | Minor | util | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze | +| [HDFS-4946](https://issues.apache.org/jira/browse/HDFS-4946) | Allow preferLocalNode in BlockPlacementPolicyDefault to be configurable | Major | namenode | James Kinley | Nathan Roberts | +| [HADOOP-11031](https://issues.apache.org/jira/browse/HADOOP-11031) | Design Document for Credential Provider API | Major | site | Larry McCay | Larry McCay | +| [HADOOP-12828](https://issues.apache.org/jira/browse/HADOOP-12828) | Print user when services are started | Trivial | . | Brandon Li | Wei-Chiu Chuang | +| [HADOOP-12794](https://issues.apache.org/jira/browse/HADOOP-12794) | Support additional compression levels for GzipCodec | Major | io | Ravi Mutyala | Ravi Mutyala | +| [HDFS-9425](https://issues.apache.org/jira/browse/HDFS-9425) | Expose number of blocks per volume as a metric | Major | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [HADOOP-12668](https://issues.apache.org/jira/browse/HADOOP-12668) | Support excluding weak Ciphers in HttpServer2 through ssl-server.conf | Critical | security | Vijay Singh | Vijay Singh | +| [HADOOP-12555](https://issues.apache.org/jira/browse/HADOOP-12555) | WASB to read credentials from a credential provider | Minor | fs/azure | Chris Nauroth | Larry McCay | +| [HDFS-8578](https://issues.apache.org/jira/browse/HDFS-8578) | On upgrade, Datanode should process all storage/data dirs in parallel | Critical | datanode | Raju Bairishetti | Vinayakumar B | +| [HADOOP-12535](https://issues.apache.org/jira/browse/HADOOP-12535) | Run FileSystem contract tests with hadoop-azure. | Major | fs/azure, test | Chris Nauroth | madhumita chakraborty | +| [HDFS-9854](https://issues.apache.org/jira/browse/HDFS-9854) | Log cipher suite negotiation more verbosely | Major | encryption | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HDFS-9843](https://issues.apache.org/jira/browse/HDFS-9843) | Document distcp options required for copying between encrypted locations | Major | distcp, documentation, encryption | Xiaoyu Yao | Xiaoyu Yao | +| [HADOOP-12824](https://issues.apache.org/jira/browse/HADOOP-12824) | Collect network and disk usage on the node running Windows | Major | . | Inigo Goiri | Inigo Goiri | +| [YARN-4720](https://issues.apache.org/jira/browse/YARN-4720) | Skip unnecessary NN operations in log aggregation | Major | . | Ming Ma | Jun Gong | +| [HDFS-9831](https://issues.apache.org/jira/browse/HDFS-9831) | Document webhdfs retry configuration keys introduced by HDFS-5219/HDFS-5122 | Major | documentation, webhdfs | Xiaoyu Yao | Xiaobing Zhou | +| [HDFS-9710](https://issues.apache.org/jira/browse/HDFS-9710) | Change DN to send block receipt IBRs in batches | Major | datanode | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze | +| [MAPREDUCE-6622](https://issues.apache.org/jira/browse/MAPREDUCE-6622) | Add capability to set JHS job cache to a task-based limit | Critical | jobhistoryserver | Ray Chiang | Ray Chiang | +| [HADOOP-12825](https://issues.apache.org/jira/browse/HADOOP-12825) | Log slow name resolutions | Major | . | Sidharta Seethana | Sidharta Seethana | +| [YARN-4671](https://issues.apache.org/jira/browse/YARN-4671) | There is no need to acquire CS lock when completing a container | Major | . | MENG DING | MENG DING | +| [HADOOP-12853](https://issues.apache.org/jira/browse/HADOOP-12853) | Change WASB documentation regarding page blob support | Minor | fs/azure | madhumita chakraborty | madhumita chakraborty | +| [HDFS-9887](https://issues.apache.org/jira/browse/HDFS-9887) | WebHdfs socket timeouts should be configurable | Major | fs, webhdfs | Austin Donnelly | Austin Donnelly | +| [HADOOP-12859](https://issues.apache.org/jira/browse/HADOOP-12859) | Disable hiding field style checks in class setters | Major | . | Kai Zheng | Kai Zheng | +| [HDFS-9534](https://issues.apache.org/jira/browse/HDFS-9534) | Add CLI command to clear storage policy from a path. | Major | tools | Chris Nauroth | Xiaobing Zhou | +| [HADOOP-12793](https://issues.apache.org/jira/browse/HADOOP-12793) | Write a new group mapping service guide | Major | documentation | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HADOOP-12470](https://issues.apache.org/jira/browse/HADOOP-12470) | In-page TOC of documentation should be automatically generated by doxia macro | Major | documentation | Masatake Iwasaki | Masatake Iwasaki | +| [HDFS-9889](https://issues.apache.org/jira/browse/HDFS-9889) | Update balancer/mover document about HDFS-6133 feature | Minor | . | Yongjun Zhang | Yongjun Zhang | +| [MAPREDUCE-6648](https://issues.apache.org/jira/browse/MAPREDUCE-6648) | Add yarn.app.mapreduce.am.log.level to mapred-default.xml | Trivial | documentation | Harsh J | Harsh J | +| [HDFS-9906](https://issues.apache.org/jira/browse/HDFS-9906) | Remove spammy log spew when a datanode is restarted | Major | namenode | Elliott Clark | Brahma Reddy Battula | +| [HADOOP-12901](https://issues.apache.org/jira/browse/HADOOP-12901) | Add warning log when KMSClientProvider cannot create a connection to the KMS server | Minor | . | Xiao Chen | Xiao Chen | +| [HADOOP-12789](https://issues.apache.org/jira/browse/HADOOP-12789) | log classpath of ApplicationClassLoader at INFO level | Minor | util | Sangjin Lee | Sangjin Lee | +| [HDFS-9882](https://issues.apache.org/jira/browse/HDFS-9882) | Add heartbeatsTotal in Datanode metrics | Minor | datanode | Hua Liu | Hua Liu | +| [HADOOP-12860](https://issues.apache.org/jira/browse/HADOOP-12860) | Expand section "Data Encryption on HTTP" in SecureMode documentation | Minor | . | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [YARN-4465](https://issues.apache.org/jira/browse/YARN-4465) | SchedulerUtils#validateRequest for Label check should happen only when nodelabel enabled | Minor | . | Bibin A Chundatt | Bibin A Chundatt | +| [HADOOP-12904](https://issues.apache.org/jira/browse/HADOOP-12904) | Update Yetus to 0.2.0 | Blocker | build | Allen Wittenauer | Allen Wittenauer | +| [HADOOP-12798](https://issues.apache.org/jira/browse/HADOOP-12798) | Update changelog and release notes (2016-03-04) | Major | documentation | Allen Wittenauer | Allen Wittenauer | +| [HADOOP-12905](https://issues.apache.org/jira/browse/HADOOP-12905) | Clean up CHANGES.txt RAT exclusions from pom.xml files. | Trivial | build | Chris Nauroth | Chris Nauroth | +| [HDFS-9927](https://issues.apache.org/jira/browse/HDFS-9927) | Document the new OIV ReverseXML processor | Minor | documentation | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HDFS-9942](https://issues.apache.org/jira/browse/HDFS-9942) | Add an HTrace span when refreshing the groups for a username | Major | . | Colin P. McCabe | Colin P. McCabe | +| [HDFS-9941](https://issues.apache.org/jira/browse/HDFS-9941) | Do not log StandbyException on NN, other minor logging fixes | Major | namenode | Arpit Agarwal | Arpit Agarwal | +| [HADOOP-12923](https://issues.apache.org/jira/browse/HADOOP-12923) | Move the test code in ipc.Client to test | Minor | ipc | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze | +| [HDFS-9405](https://issues.apache.org/jira/browse/HDFS-9405) | Warmup NameNode EDEK caches in background thread | Major | encryption, namenode | Zhe Zhang | Xiao Chen | +| [HDFS-9951](https://issues.apache.org/jira/browse/HDFS-9951) | Use string constants for XML tags in OfflineImageReconstructor | Minor | . | Yiqun Lin | Yiqun Lin | +| [HADOOP-12947](https://issues.apache.org/jira/browse/HADOOP-12947) | Update documentation Hadoop Groups Mapping to add static group mapping, negative cache | Minor | documentation, security | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [YARN-4117](https://issues.apache.org/jira/browse/YARN-4117) | End to end unit test with mini YARN cluster for AMRMProxy Service | Major | nodemanager, resourcemanager | Kishore Chaliparambil | Giovanni Matteo Fumarola | +| [HADOOP-10965](https://issues.apache.org/jira/browse/HADOOP-10965) | Print fully qualified path in CommandWithDestination error messages | Minor | . | André Kelpe | John Zhuge | +| [MAPREDUCE-6663](https://issues.apache.org/jira/browse/MAPREDUCE-6663) | [NNBench] Refactor nnbench as a Tool implementation. | Major | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [HADOOP-12886](https://issues.apache.org/jira/browse/HADOOP-12886) | Exclude weak ciphers in SSLFactory through ssl-server.xml | Major | . | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [YARN-4884](https://issues.apache.org/jira/browse/YARN-4884) | Fix missing documentation about rmadmin command regarding node labels | Minor | . | Kai Sasaki | Kai Sasaki | +| [HADOOP-12916](https://issues.apache.org/jira/browse/HADOOP-12916) | Allow RPC scheduler/callqueue backoff using response times | Major | ipc | Xiaoyu Yao | Xiaoyu Yao | +| [HADOOP-12950](https://issues.apache.org/jira/browse/HADOOP-12950) | ShutdownHookManager should have a timeout for each of the Registered shutdown hook | Major | . | Xiaoyu Yao | Xiaoyu Yao | +| [HADOOP-11661](https://issues.apache.org/jira/browse/HADOOP-11661) | Deprecate FileUtil#copyMerge | Major | util | Brahma Reddy Battula | Brahma Reddy Battula | +| [HADOOP-11687](https://issues.apache.org/jira/browse/HADOOP-11687) | Ignore x-\* and response headers when copying an Amazon S3 object | Major | fs/s3 | Denis Jannot | Harsh J | +| [HADOOP-11212](https://issues.apache.org/jira/browse/HADOOP-11212) | NetUtils.wrapException to handle SocketException explicitly | Major | util | Steve Loughran | Steve Loughran | +| [HADOOP-12672](https://issues.apache.org/jira/browse/HADOOP-12672) | RPC timeout should not override IPC ping interval | Major | ipc | Masatake Iwasaki | Masatake Iwasaki | +| [HDFS-10235](https://issues.apache.org/jira/browse/HDFS-10235) | [NN UI] Last contact for Live Nodes should be relative time | Major | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [HADOOP-8813](https://issues.apache.org/jira/browse/HADOOP-8813) | RPC Server and Client classes need InterfaceAudience and InterfaceStability annotations | Trivial | ipc | Brandon Li | Brandon Li | +| [YARN-4756](https://issues.apache.org/jira/browse/YARN-4756) | Unnecessary wait in Node Status Updater during reboot | Major | . | Eric Badger | Eric Badger | +| [HADOOP-12951](https://issues.apache.org/jira/browse/HADOOP-12951) | Improve documentation on KMS ACLs and delegation tokens | Major | . | Xiao Chen | Xiao Chen | +| [HADOOP-12994](https://issues.apache.org/jira/browse/HADOOP-12994) | Specify PositionedReadable, add contract tests, fix problems | Major | fs | Steve Loughran | Steve Loughran | +| [YARN-4630](https://issues.apache.org/jira/browse/YARN-4630) | Remove useless boxing/unboxing code | Minor | yarn | Kousuke Saruta | Kousuke Saruta | +| [HDFS-10279](https://issues.apache.org/jira/browse/HDFS-10279) | Improve validation of the configured number of tolerated failed volumes | Major | . | Yiqun Lin | Yiqun Lin | +| [HADOOP-12822](https://issues.apache.org/jira/browse/HADOOP-12822) | Change "Metrics intern cache overflow" log level from WARN to INFO | Minor | metrics | Akira Ajisaka | Andras Bokor | +| [HADOOP-12969](https://issues.apache.org/jira/browse/HADOOP-12969) | Mark IPC.Client and IPC.Server as @Public, @Evolving | Minor | ipc | Xiaobing Zhou | Xiaobing Zhou | +| [HADOOP-12963](https://issues.apache.org/jira/browse/HADOOP-12963) | Allow using path style addressing for accessing the s3 endpoint | Minor | fs/s3 | Andrew Baptist | Stephen Montgomery | +| [HDFS-10280](https://issues.apache.org/jira/browse/HDFS-10280) | Document new dfsadmin command -evictWriters | Minor | documentation | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HDFS-10292](https://issues.apache.org/jira/browse/HDFS-10292) | Add block id when client got Unable to close file exception | Minor | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [HDFS-9412](https://issues.apache.org/jira/browse/HDFS-9412) | getBlocks occupies FSLock and takes too long to complete | Major | . | He Tianyi | He Tianyi | +| [HDFS-10302](https://issues.apache.org/jira/browse/HDFS-10302) | BlockPlacementPolicyDefault should use default replication considerload value | Trivial | . | Yiqun Lin | Yiqun Lin | +| [HDFS-10264](https://issues.apache.org/jira/browse/HDFS-10264) | Logging improvements in FSImageFormatProtobuf.Saver | Major | namenode | Konstantin Shvachko | Xiaobing Zhou | +| [HADOOP-12985](https://issues.apache.org/jira/browse/HADOOP-12985) | Support MetricsSource interface for DecayRpcScheduler Metrics | Major | . | Xiaoyu Yao | Xiaoyu Yao | +| [HDFS-9894](https://issues.apache.org/jira/browse/HDFS-9894) | Add unsetStoragePolicy API to FileContext/AbstractFileSystem and derivatives | Major | . | Xiaobing Zhou | Xiaobing Zhou | +| [HADOOP-12891](https://issues.apache.org/jira/browse/HADOOP-12891) | S3AFileSystem should configure Multipart Copy threshold and chunk size | Major | fs/s3 | Andrew Olson | Andrew Olson | +| [HADOOP-13033](https://issues.apache.org/jira/browse/HADOOP-13033) | Add missing Javadoc enries to Interns.java | Minor | metrics | Andras Bokor | Andras Bokor | +| [HDFS-10298](https://issues.apache.org/jira/browse/HDFS-10298) | Document the usage of distcp -diff option | Major | distcp, documentation | Akira Ajisaka | Takashi Ohnishi | +| [HADOOP-13039](https://issues.apache.org/jira/browse/HADOOP-13039) | Add documentation for configuration property ipc.maximum.data.length for controlling maximum RPC message size. | Major | documentation | Chris Nauroth | Mingliang Liu | +| [HDFS-10330](https://issues.apache.org/jira/browse/HDFS-10330) | Add Corrupt Blocks Information in Metasave Output | Major | . | Kuhu Shukla | Kuhu Shukla | +| [HADOOP-5470](https://issues.apache.org/jira/browse/HADOOP-5470) | RunJar.unJar() should write the last modified time found in the jar entry to the uncompressed file | Minor | util | Colin Evans | Andras Bokor | +| [HDFS-3702](https://issues.apache.org/jira/browse/HDFS-3702) | Add an option for NOT writing the blocks locally if there is a datanode on the same box as the client | Minor | hdfs-client | Nicolas Liochon | Lei (Eddy) Xu | +| [HDFS-10297](https://issues.apache.org/jira/browse/HDFS-10297) | Increase default balance bandwidth and concurrent moves | Minor | balancer & mover | John Zhuge | John Zhuge | +| [HADOOP-12957](https://issues.apache.org/jira/browse/HADOOP-12957) | Limit the number of outstanding async calls | Major | ipc | Xiaobing Zhou | Xiaobing Zhou | +| [HDFS-9902](https://issues.apache.org/jira/browse/HDFS-9902) | Support different values of dfs.datanode.du.reserved per storage type | Major | datanode | Pan Yuxuan | Brahma Reddy Battula | +| [HADOOP-13103](https://issues.apache.org/jira/browse/HADOOP-13103) | Group resolution from LDAP may fail on javax.naming.ServiceUnavailableException | Minor | security | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze | +| [MAPREDUCE-6678](https://issues.apache.org/jira/browse/MAPREDUCE-6678) | Allow ShuffleHandler readahead without drop-behind | Major | nodemanager | Nathan Roberts | Nathan Roberts | +| [HADOOP-12982](https://issues.apache.org/jira/browse/HADOOP-12982) | Document missing S3A and S3 properties | Minor | documentation, fs/s3, tools | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [YARN-4995](https://issues.apache.org/jira/browse/YARN-4995) | FairScheduler: Display per-queue demand on the scheduler page | Minor | . | xupeng | xupeng | +| [HADOOP-12868](https://issues.apache.org/jira/browse/HADOOP-12868) | Fix hadoop-openstack undeclared and unused dependencies | Major | tools | Allen Wittenauer | Masatake Iwasaki | +| [HADOOP-12971](https://issues.apache.org/jira/browse/HADOOP-12971) | FileSystemShell doc should explain relative path | Critical | documentation | John Zhuge | John Zhuge | +| [HADOOP-13148](https://issues.apache.org/jira/browse/HADOOP-13148) | TestDistCpViewFs to include IOExceptions in test error reports | Minor | tools/distcp | Steve Loughran | Steve Loughran | +| [HADOOP-13146](https://issues.apache.org/jira/browse/HADOOP-13146) | Refactor RetryInvocationHandler | Minor | io | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze | +| [HDFS-10383](https://issues.apache.org/jira/browse/HDFS-10383) | Safely close resources in DFSTestUtil | Major | test | Mingliang Liu | Mingliang Liu | +| [YARN-4002](https://issues.apache.org/jira/browse/YARN-4002) | make ResourceTrackerService.nodeHeartbeat more concurrent | Critical | . | Hong Zhiguo | Hong Zhiguo | +| [HDFS-10417](https://issues.apache.org/jira/browse/HDFS-10417) | Improve error message from checkBlockLocalPathAccess | Minor | datanode | Tianyin Xu | Tianyin Xu | +| [HADOOP-13168](https://issues.apache.org/jira/browse/HADOOP-13168) | Support Future.get with timeout in ipc async calls | Major | ipc | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze | +| [HADOOP-13145](https://issues.apache.org/jira/browse/HADOOP-13145) | In DistCp, prevent unnecessary getFileStatus call when not preserving metadata. | Major | tools/distcp | Chris Nauroth | Chris Nauroth | +| [HADOOP-13198](https://issues.apache.org/jira/browse/HADOOP-13198) | Add support for OWASP's dependency-check | Minor | build, security | Mike Yoder | Mike Yoder | +| [HDFS-10217](https://issues.apache.org/jira/browse/HDFS-10217) | show 'blockScheduled' tooltip in datanodes table. | Major | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [HADOOP-13199](https://issues.apache.org/jira/browse/HADOOP-13199) | Add doc for distcp -filters | Trivial | documentation | John Zhuge | John Zhuge | +| [HADOOP-13193](https://issues.apache.org/jira/browse/HADOOP-13193) | Upgrade to Apache Yetus 0.3.0 | Major | documentation, test | Allen Wittenauer | Kengo Seki | +| [HDFS-10341](https://issues.apache.org/jira/browse/HDFS-10341) | Add a metric to expose the timeout number of pending replication blocks | Major | . | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-13105](https://issues.apache.org/jira/browse/HADOOP-13105) | Support timeouts in LDAP queries in LdapGroupsMapping. | Major | security | Chris Nauroth | Mingliang Liu | +| [HADOOP-12807](https://issues.apache.org/jira/browse/HADOOP-12807) | S3AFileSystem should read AWS credentials from environment variables | Minor | fs/s3 | Tobin Baker | Tobin Baker | +| [MAPREDUCE-5044](https://issues.apache.org/jira/browse/MAPREDUCE-5044) | Have AM trigger jstack on task attempts that timeout before killing them | Major | mr-am | Jason Lowe | Eric Payne | +| [HADOOP-10048](https://issues.apache.org/jira/browse/HADOOP-10048) | LocalDirAllocator should avoid holding locks while accessing the filesystem | Major | . | Jason Lowe | Jason Lowe | +| [MAPREDUCE-6714](https://issues.apache.org/jira/browse/MAPREDUCE-6714) | Refactor UncompressedSplitLineReader.fillBuffer() | Major | . | Daniel Templeton | Daniel Templeton | +| [HADOOP-12943](https://issues.apache.org/jira/browse/HADOOP-12943) | Add -w -r options in dfs -test command | Major | fs, scripts, tools | Weiwei Yang | Weiwei Yang | +| [HDFS-10493](https://issues.apache.org/jira/browse/HDFS-10493) | Add links to datanode web UI in namenode datanodes page | Major | namenode, ui | Weiwei Yang | Weiwei Yang | +| [HADOOP-13296](https://issues.apache.org/jira/browse/HADOOP-13296) | Cleanup javadoc for Path | Minor | documentation | Daniel Templeton | Daniel Templeton | +| [HDFS-7597](https://issues.apache.org/jira/browse/HDFS-7597) | DelegationTokenIdentifier should cache the TokenIdentifier to UGI mapping | Critical | webhdfs | Daryn Sharp | Daryn Sharp | +| [HADOOP-13307](https://issues.apache.org/jira/browse/HADOOP-13307) | add rsync to Dockerfile so that precommit archive works | Trivial | build | Allen Wittenauer | Allen Wittenauer | +| [HADOOP-13067](https://issues.apache.org/jira/browse/HADOOP-13067) | cleanup the dockerfile | Major | . | Allen Wittenauer | Allen Wittenauer | +| [HADOOP-13227](https://issues.apache.org/jira/browse/HADOOP-13227) | AsyncCallHandler should use an event driven architecture to handle async calls | Major | io, ipc | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze | +| [HADOOP-13263](https://issues.apache.org/jira/browse/HADOOP-13263) | Reload cached groups in background after expiry | Major | . | Stephen O'Donnell | Stephen O'Donnell | +| [HDFS-10440](https://issues.apache.org/jira/browse/HDFS-10440) | Improve DataNode web UI | Major | datanode, ui | Weiwei Yang | Weiwei Yang | +| [HADOOP-13239](https://issues.apache.org/jira/browse/HADOOP-13239) | Deprecate s3:// in branch-2 | Major | fs/s3 | Mingliang Liu | Mingliang Liu | +| [HDFS-10582](https://issues.apache.org/jira/browse/HDFS-10582) | Change deprecated configuration fs.checkpoint.dir to dfs.namenode.checkpoint.dir in HDFS Commands Doc | Minor | documentation | Pan Yuxuan | Pan Yuxuan | +| [HDFS-10488](https://issues.apache.org/jira/browse/HDFS-10488) | Update WebHDFS documentation regarding CREATE and MKDIR default permissions | Minor | documentation, webhdfs | Wellington Chevreuil | Wellington Chevreuil | +| [HDFS-10300](https://issues.apache.org/jira/browse/HDFS-10300) | TestDistCpSystem should share MiniDFSCluster | Trivial | test | John Zhuge | John Zhuge | +| [HADOOP-13290](https://issues.apache.org/jira/browse/HADOOP-13290) | Appropriate use of generics in FairCallQueue | Major | ipc | Konstantin Shvachko | Jonathan Hung | +| [HADOOP-13289](https://issues.apache.org/jira/browse/HADOOP-13289) | Remove unused variables in TestFairCallQueue | Minor | test | Konstantin Shvachko | Ye Zhou | +| [HDFS-10628](https://issues.apache.org/jira/browse/HDFS-10628) | Log HDFS Balancer exit message to its own log | Minor | balancer & mover | Jiayi Zhou | Jiayi Zhou | +| [HADOOP-13298](https://issues.apache.org/jira/browse/HADOOP-13298) | Fix the leftover L&N files in hadoop-build-tools/src/main/resources/META-INF/ | Minor | . | Xiao Chen | Tsuyoshi Ozawa | +| [YARN-4883](https://issues.apache.org/jira/browse/YARN-4883) | Make consistent operation name in AdminService | Minor | resourcemanager | Kai Sasaki | Kai Sasaki | +| [YARN-1126](https://issues.apache.org/jira/browse/YARN-1126) | Add validation of users input nodes-states options to nodes CLI | Major | . | Wei Yan | Wei Yan | +| [HDFS-10287](https://issues.apache.org/jira/browse/HDFS-10287) | MiniDFSCluster should implement AutoCloseable | Minor | test | John Zhuge | Andras Bokor | +| [HDFS-10225](https://issues.apache.org/jira/browse/HDFS-10225) | DataNode hot swap drives should disallow storage type changes. | Major | datanode | Lei (Eddy) Xu | Lei (Eddy) Xu | +| [HDFS-10660](https://issues.apache.org/jira/browse/HDFS-10660) | Expose storage policy apis via HDFSAdmin interface | Major | . | Rakesh R | Rakesh R | +| [HDFS-9937](https://issues.apache.org/jira/browse/HDFS-9937) | Update dfsadmin command line help and HdfsQuotaAdminGuide | Minor | . | Wei-Chiu Chuang | Kai Sasaki | +| [HDFS-10667](https://issues.apache.org/jira/browse/HDFS-10667) | Report more accurate info about data corruption location | Major | datanode, hdfs | Yongjun Zhang | Yuanbo Liu | +| [HDFS-10676](https://issues.apache.org/jira/browse/HDFS-10676) | Add namenode metric to measure time spent in generating EDEKs | Major | namenode | Hanisha Koneru | Hanisha Koneru | +| [MAPREDUCE-6746](https://issues.apache.org/jira/browse/MAPREDUCE-6746) | Replace org.apache.commons.io.Charsets with java.nio.charset.StandardCharsets | Minor | . | Vincent Poon | Vincent Poon | +| [HDFS-10703](https://issues.apache.org/jira/browse/HDFS-10703) | HA NameNode Web UI should show last checkpoint time | Minor | ui | John Zhuge | John Zhuge | +| [MAPREDUCE-6729](https://issues.apache.org/jira/browse/MAPREDUCE-6729) | Accurately compute the test execute time in DFSIO | Minor | benchmarks, performance, test | mingleizhang | mingleizhang | +| [HADOOP-13444](https://issues.apache.org/jira/browse/HADOOP-13444) | Replace org.apache.commons.io.Charsets with java.nio.charset.StandardCharsets | Minor | . | Vincent Poon | Vincent Poon | +| [YARN-5456](https://issues.apache.org/jira/browse/YARN-5456) | container-executor support for FreeBSD, NetBSD, and others if conf path is absolute | Major | nodemanager, security | Allen Wittenauer | Allen Wittenauer | +| [MAPREDUCE-6730](https://issues.apache.org/jira/browse/MAPREDUCE-6730) | Use StandardCharsets instead of String overload in TextOutputFormat | Minor | . | Sahil Kang | Sahil Kang | +| [HDFS-10707](https://issues.apache.org/jira/browse/HDFS-10707) | Replace org.apache.commons.io.Charsets with java.nio.charset.StandardCharsets | Minor | . | Vincent Poon | Vincent Poon | +| [HADOOP-13442](https://issues.apache.org/jira/browse/HADOOP-13442) | Optimize UGI group lookups | Major | . | Daryn Sharp | Daryn Sharp | +| [HADOOP-13466](https://issues.apache.org/jira/browse/HADOOP-13466) | Add an AutoCloseableLock class | Major | . | Chen Liang | Chen Liang | +| [YARN-5483](https://issues.apache.org/jira/browse/YARN-5483) | Optimize RMAppAttempt#pullJustFinishedContainers | Major | . | sandflee | sandflee | +| [HADOOP-13190](https://issues.apache.org/jira/browse/HADOOP-13190) | Mention LoadBalancingKMSClientProvider in KMS HA documentation | Major | documentation, kms | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HDFS-10677](https://issues.apache.org/jira/browse/HDFS-10677) | Über-jira: Enhancements to NNThroughputBenchmark tool | Major | benchmarks, tools | Mingliang Liu | Mingliang Liu | +| [HDFS-10342](https://issues.apache.org/jira/browse/HDFS-10342) | BlockManager#createLocatedBlocks should not check corrupt replicas if none are corrupt | Major | hdfs | Daryn Sharp | Kuhu Shukla | +| [HDFS-10682](https://issues.apache.org/jira/browse/HDFS-10682) | Replace FsDatasetImpl object lock with a separate lock object | Major | datanode | Chen Liang | Chen Liang | +| [HADOOP-13503](https://issues.apache.org/jira/browse/HADOOP-13503) | Improve SaslRpcClient failure logging | Major | security | Xiaobing Zhou | Xiaobing Zhou | +| [HADOOP-13527](https://issues.apache.org/jira/browse/HADOOP-13527) | Add Spark to CallerContext LimitedPrivate scope | Minor | ipc | Weiqing Yang | Weiqing Yang | +| [MAPREDUCE-6587](https://issues.apache.org/jira/browse/MAPREDUCE-6587) | Remove unused params in connection-related methods of Fetcher | Minor | . | Yiqun Lin | Yiqun Lin | +| [HADOOP-13538](https://issues.apache.org/jira/browse/HADOOP-13538) | Deprecate getInstance and initialize methods with Path in TrashPolicy | Minor | . | Yiqun Lin | Yiqun Lin | +| [HDFS-8986](https://issues.apache.org/jira/browse/HDFS-8986) | Add option to -du to calculate directory space usage excluding snapshots | Major | snapshots | Gautam Gopalakrishnan | Xiao Chen | +| [HDFS-10798](https://issues.apache.org/jira/browse/HDFS-10798) | Make the threshold of reporting FSNamesystem lock contention configurable | Major | logging, namenode | Zhe Zhang | Erik Krogen | +| [YARN-5550](https://issues.apache.org/jira/browse/YARN-5550) | TestYarnCLI#testGetContainers should format according to CONTAINER\_PATTERN | Minor | client, test | Jonathan Hung | Jonathan Hung | +| [HDFS-10814](https://issues.apache.org/jira/browse/HDFS-10814) | Add assertion for getNumEncryptionZones when no EZ is created | Minor | test | Vinitha Reddy Gankidi | Vinitha Reddy Gankidi | +| [HDFS-10817](https://issues.apache.org/jira/browse/HDFS-10817) | Add Logging for Long-held NN Read Locks | Major | logging, namenode | Erik Krogen | Erik Krogen | +| [HADOOP-13465](https://issues.apache.org/jira/browse/HADOOP-13465) | Design Server.Call to be extensible for unified call queue | Major | ipc | Daryn Sharp | Daryn Sharp | +| [HDFS-10833](https://issues.apache.org/jira/browse/HDFS-10833) | Fix JSON errors in WebHDFS.md examples | Trivial | documentation | Andrew Wang | Andrew Wang | +| [HDFS-10742](https://issues.apache.org/jira/browse/HDFS-10742) | Measure lock time in FsDatasetImpl | Major | datanode | Chen Liang | Chen Liang | +| [HDFS-10831](https://issues.apache.org/jira/browse/HDFS-10831) | Add log when URLConnectionFactory.openConnection failed | Minor | webhdfs | yunjiong zhao | yunjiong zhao | +| [HADOOP-13598](https://issues.apache.org/jira/browse/HADOOP-13598) | Add eol=lf for unix format files in .gitattributes | Major | . | Akira Ajisaka | Yiqun Lin | +| [HADOOP-13580](https://issues.apache.org/jira/browse/HADOOP-13580) | If user is unauthorized, log "unauthorized" instead of "Invalid signed text:" | Minor | security | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HDFS-10489](https://issues.apache.org/jira/browse/HDFS-10489) | Deprecate dfs.encryption.key.provider.uri for HDFS encryption zones | Minor | . | Xiao Chen | Xiao Chen | +| [YARN-5540](https://issues.apache.org/jira/browse/YARN-5540) | scheduler spends too much time looking at empty priorities | Major | capacity scheduler, fairscheduler, resourcemanager | Nathan Roberts | Jason Lowe | +| [HADOOP-13169](https://issues.apache.org/jira/browse/HADOOP-13169) | Randomize file list in SimpleCopyListing | Minor | tools/distcp | Rajesh Balamohan | Rajesh Balamohan | +| [HDFS-10875](https://issues.apache.org/jira/browse/HDFS-10875) | Optimize du -x to cache intermediate result | Major | snapshots | Xiao Chen | Xiao Chen | +| [YARN-4591](https://issues.apache.org/jira/browse/YARN-4591) | YARN Web UIs should provide a robots.txt | Trivial | . | Lars Francke | Sidharta Seethana | +| [YARN-5622](https://issues.apache.org/jira/browse/YARN-5622) | TestYarnCLI.testGetContainers fails due to mismatched date formats | Minor | . | Eric Badger | Eric Badger | +| [HDFS-10876](https://issues.apache.org/jira/browse/HDFS-10876) | Dispatcher#dispatch should log IOException stacktrace | Trivial | balancer & mover | Wei-Chiu Chuang | Manoj Govindassamy | +| [YARN-3692](https://issues.apache.org/jira/browse/YARN-3692) | Allow REST API to set a user generated message when killing an application | Major | . | Rajat Jain | Rohith Sharma K S | +| [HDFS-10869](https://issues.apache.org/jira/browse/HDFS-10869) | Remove the unused method InodeId#checkId() | Major | namenode | Jagadesh Kiran N | Jagadesh Kiran N | +| [YARN-3877](https://issues.apache.org/jira/browse/YARN-3877) | YarnClientImpl.submitApplication swallows exceptions | Minor | client | Steve Loughran | Varun Saxena | +| [HADOOP-13658](https://issues.apache.org/jira/browse/HADOOP-13658) | Replace config key literal strings with config key names I: hadoop common | Minor | conf | Chen Liang | Chen Liang | +| [HADOOP-13537](https://issues.apache.org/jira/browse/HADOOP-13537) | Support external calls in the RPC call queue | Major | ipc | Daryn Sharp | Daryn Sharp | +| [HADOOP-13317](https://issues.apache.org/jira/browse/HADOOP-13317) | Add logs to KMS server-side to improve supportability | Minor | kms | Xiao Chen | Suraj Acharya | +| [HDFS-10940](https://issues.apache.org/jira/browse/HDFS-10940) | Reduce performance penalty of block caching when not used | Major | namenode | Daryn Sharp | Daryn Sharp | +| [YARN-4855](https://issues.apache.org/jira/browse/YARN-4855) | Should check if node exists when replace nodelabels | Minor | . | Tao Jie | Tao Jie | +| [HDFS-10690](https://issues.apache.org/jira/browse/HDFS-10690) | Optimize insertion/removal of replica in ShortCircuitCache | Major | hdfs-client | Fenghua Hu | Fenghua Hu | +| [HADOOP-13685](https://issues.apache.org/jira/browse/HADOOP-13685) | Document -safely option of rm command. | Major | . | Xiaoyu Yao | Xiaoyu Yao | +| [MAPREDUCE-6741](https://issues.apache.org/jira/browse/MAPREDUCE-6741) | add MR support to redact job conf properties | Major | mrv2 | Haibo Chen | Haibo Chen | +| [HDFS-10963](https://issues.apache.org/jira/browse/HDFS-10963) | Reduce log level when network topology cannot find enough datanodes. | Minor | . | Xiao Chen | Xiao Chen | +| [HADOOP-13323](https://issues.apache.org/jira/browse/HADOOP-13323) | Downgrade stack trace on FS load from Warn to debug | Minor | fs | Steve Loughran | Steve Loughran | +| [HADOOP-13150](https://issues.apache.org/jira/browse/HADOOP-13150) | Avoid use of toString() in output of HDFS ACL shell commands. | Minor | . | Chris Nauroth | Chris Nauroth | +| [HADOOP-13689](https://issues.apache.org/jira/browse/HADOOP-13689) | Do not attach javadoc and sources jars during non-dist build | Major | . | Andrew Wang | Andrew Wang | +| [HADOOP-13641](https://issues.apache.org/jira/browse/HADOOP-13641) | Update UGI#spawnAutoRenewalThreadForUserCreds to reduce indentation | Minor | . | Xiao Chen | Huafeng Wang | +| [HDFS-10933](https://issues.apache.org/jira/browse/HDFS-10933) | Refactor TestFsck | Minor | . | Takanobu Asanuma | Takanobu Asanuma | +| [HADOOP-13684](https://issues.apache.org/jira/browse/HADOOP-13684) | Snappy may complain Hadoop is built without snappy if libhadoop is not found. | Minor | . | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HADOOP-13698](https://issues.apache.org/jira/browse/HADOOP-13698) | Document caveat for KeyShell when underlying KeyProvider does not delete a key | Minor | documentation, kms | Xiao Chen | Xiao Chen | +| [HDFS-10789](https://issues.apache.org/jira/browse/HDFS-10789) | Route webhdfs through the RPC call queue | Major | ipc, webhdfs | Daryn Sharp | Daryn Sharp | +| [HDFS-10903](https://issues.apache.org/jira/browse/HDFS-10903) | Replace config key literal strings with config key names II: hadoop hdfs | Minor | . | Mingliang Liu | Chen Liang | +| [HADOOP-13710](https://issues.apache.org/jira/browse/HADOOP-13710) | Supress CachingGetSpaceUsed from logging interrupted exception stacktrace | Minor | fs | Wei-Chiu Chuang | Hanisha Koneru | +| [YARN-5599](https://issues.apache.org/jira/browse/YARN-5599) | Publish AM launch command to ATS | Major | . | Daniel Templeton | Rohith Sharma K S | +| [HDFS-11012](https://issues.apache.org/jira/browse/HDFS-11012) | Unnecessary INFO logging on DFSClients for InvalidToken | Minor | fs | Harsh J | Harsh J | +| [HDFS-11003](https://issues.apache.org/jira/browse/HDFS-11003) | Expose "XmitsInProgress" through DataNodeMXBean | Major | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [HADOOP-13724](https://issues.apache.org/jira/browse/HADOOP-13724) | Fix a few typos in site markdown documents | Minor | documentation | Andrew Wang | Ding Fei | +| [HDFS-11009](https://issues.apache.org/jira/browse/HDFS-11009) | Add a tool to reconstruct block meta file from CLI | Major | datanode | Xiao Chen | Xiao Chen | +| [HDFS-9480](https://issues.apache.org/jira/browse/HDFS-9480) | Expose nonDfsUsed via StorageTypeStats | Major | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [HADOOP-12082](https://issues.apache.org/jira/browse/HADOOP-12082) | Support multiple authentication schemes via AuthenticationFilter | Major | security | Hrishikesh Gadre | Hrishikesh Gadre | +| [HADOOP-13669](https://issues.apache.org/jira/browse/HADOOP-13669) | KMS Server should log exceptions before throwing | Major | kms | Xiao Chen | Suraj Acharya | +| [HADOOP-13502](https://issues.apache.org/jira/browse/HADOOP-13502) | Split fs.contract.is-blobstore flag into more descriptive flags for use by contract tests. | Minor | test | Chris Nauroth | Chris Nauroth | +| [HADOOP-13017](https://issues.apache.org/jira/browse/HADOOP-13017) | Implementations of InputStream.read(buffer, offset, bytes) to exit 0 if bytes==0 | Major | fs, io | Steve Loughran | Steve Loughran | +| [HDFS-11055](https://issues.apache.org/jira/browse/HDFS-11055) | Update default-log4j.properties for httpfs to imporve test logging | Major | httpfs, test | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [YARN-4963](https://issues.apache.org/jira/browse/YARN-4963) | capacity scheduler: Make number of OFF\_SWITCH assignments per heartbeat configurable | Major | capacityscheduler | Nathan Roberts | Nathan Roberts | +| [HDFS-11047](https://issues.apache.org/jira/browse/HDFS-11047) | Remove deep copies of FinalizedReplica to alleviate heap consumption on DataNode | Major | datanode | Xiaobing Zhou | Xiaobing Zhou | +| [MAPREDUCE-6799](https://issues.apache.org/jira/browse/MAPREDUCE-6799) | Document mapreduce.jobhistory.webapp.https.address in mapred-default.xml | Minor | documentation, jobhistoryserver | Akira Ajisaka | Yiqun Lin | +| [HADOOP-12325](https://issues.apache.org/jira/browse/HADOOP-12325) | RPC Metrics : Add the ability track and log slow RPCs | Major | ipc, metrics | Anu Engineer | Anu Engineer | +| [HDFS-11074](https://issues.apache.org/jira/browse/HDFS-11074) | Remove unused method FsDatasetSpi#getFinalizedBlocksOnPersistentStorage | Major | datanode | Arpit Agarwal | Hanisha Koneru | +| [MAPREDUCE-6795](https://issues.apache.org/jira/browse/MAPREDUCE-6795) | Update the document for JobConf#setNumReduceTasks | Major | documentation | Akira Ajisaka | Yiqun Lin | +| [HADOOP-13603](https://issues.apache.org/jira/browse/HADOOP-13603) | Ignore package line length checkstyle rule | Major | build | Shane Kumpf | Shane Kumpf | +| [HADOOP-13583](https://issues.apache.org/jira/browse/HADOOP-13583) | Incorporate checkcompatibility script which runs Java API Compliance Checker | Major | scripts | Andrew Wang | Andrew Wang | +| [HDFS-11080](https://issues.apache.org/jira/browse/HDFS-11080) | Update HttpFS to use ConfigRedactor | Major | . | Sean Mackrory | Sean Mackrory | +| [YARN-5697](https://issues.apache.org/jira/browse/YARN-5697) | Use CliParser to parse options in RMAdminCLI | Major | . | Tao Jie | Tao Jie | +| [HADOOP-12453](https://issues.apache.org/jira/browse/HADOOP-12453) | Support decoding KMS Delegation Token with its own Identifier | Major | kms, security | Xiaoyu Yao | Xiaoyu Yao | +| [HADOOP-7930](https://issues.apache.org/jira/browse/HADOOP-7930) | Kerberos relogin interval in UserGroupInformation should be configurable | Major | security | Alejandro Abdelnur | Robert Kanter | +| [YARN-5720](https://issues.apache.org/jira/browse/YARN-5720) | Update document for "rmadmin -replaceLabelOnNode" | Minor | . | Tao Jie | Tao Jie | +| [HADOOP-13782](https://issues.apache.org/jira/browse/HADOOP-13782) | Make MutableRates metrics thread-local write, aggregate-on-read | Major | metrics | Erik Krogen | Erik Krogen | +| [HADOOP-13590](https://issues.apache.org/jira/browse/HADOOP-13590) | Retry until TGT expires even if the UGI renewal thread encountered exception | Major | security | Xiao Chen | Xiao Chen | +| [HDFS-11120](https://issues.apache.org/jira/browse/HDFS-11120) | TestEncryptionZones should waitActive | Minor | test | Xiao Chen | John Zhuge | +| [HADOOP-13720](https://issues.apache.org/jira/browse/HADOOP-13720) | Add more info to the msgs printed in AbstractDelegationTokenSecretManager for better supportability | Trivial | common, security | Yongjun Zhang | Yongjun Zhang | +| [HDFS-10941](https://issues.apache.org/jira/browse/HDFS-10941) | Improve BlockManager#processMisReplicatesAsync log | Major | namenode | Xiaoyu Yao | Chen Liang | +| [HADOOP-13810](https://issues.apache.org/jira/browse/HADOOP-13810) | Add a test to verify that Configuration handles &-encoded characters | Minor | test | Steve Loughran | Steve Loughran | +| [HADOOP-13742](https://issues.apache.org/jira/browse/HADOOP-13742) | Expose "NumOpenConnectionsPerUser" as a metric | Major | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [HADOOP-13646](https://issues.apache.org/jira/browse/HADOOP-13646) | Remove outdated overview.html | Minor | . | Akira Ajisaka | Brahma Reddy Battula | +| [HADOOP-13166](https://issues.apache.org/jira/browse/HADOOP-13166) | add getFileStatus("/") test to AbstractContractGetFileStatusTest | Minor | fs, test | Steve Loughran | Steve Loughran | +| [HADOOP-11603](https://issues.apache.org/jira/browse/HADOOP-11603) | Metric Snapshot log can be changed #MetricsSystemImpl.java since all the services will be initialized | Minor | metrics | Brahma Reddy Battula | Brahma Reddy Battula | +| [HADOOP-10776](https://issues.apache.org/jira/browse/HADOOP-10776) | Open up already widely-used APIs for delegation-token fetching & renewal to ecosystem projects | Blocker | . | Robert Joseph Evans | Vinod Kumar Vavilapalli | +| [HDFS-11175](https://issues.apache.org/jira/browse/HDFS-11175) | Document uppercase key names are not supported in TransparentEncryption.md | Minor | documentation | Yuanbo Liu | Yiqun Lin | +| [HADOOP-13790](https://issues.apache.org/jira/browse/HADOOP-13790) | Make qbt script executable | Trivial | scripts | Andrew Wang | Andrew Wang | +| [HDFS-8674](https://issues.apache.org/jira/browse/HDFS-8674) | Improve performance of postponed block scans | Critical | namenode | Daryn Sharp | Daryn Sharp | +| [HDFS-10581](https://issues.apache.org/jira/browse/HDFS-10581) | Hide redundant table on NameNode WebUI when no nodes are decomissioning | Trivial | hdfs, ui | Weiwei Yang | Weiwei Yang | +| [HDFS-11217](https://issues.apache.org/jira/browse/HDFS-11217) | Annotate NameNode and DataNode MXBean interfaces as Private/Stable | Major | . | Akira Ajisaka | Jagadesh Kiran N | +| [HADOOP-13900](https://issues.apache.org/jira/browse/HADOOP-13900) | Remove snapshot version of SDK dependency from Azure Data Lake Store File System | Major | fs/adl | Vishwajeet Dusane | Vishwajeet Dusane | +| [HDFS-11249](https://issues.apache.org/jira/browse/HDFS-11249) | Redundant toString() in DFSConfigKeys.java | Trivial | . | Akira Ajisaka | Jagadesh Kiran N | +| [YARN-5882](https://issues.apache.org/jira/browse/YARN-5882) | Test only changes from YARN-4126 | Major | . | Andrew Wang | Jian He | +| [HDFS-11262](https://issues.apache.org/jira/browse/HDFS-11262) | Remove unused variables in FSImage.java | Trivial | . | Akira Ajisaka | Jagadesh Kiran N | +| [YARN-4994](https://issues.apache.org/jira/browse/YARN-4994) | Use MiniYARNCluster with try-with-resources in tests | Trivial | test | Andras Bokor | Andras Bokor | +| [YARN-5709](https://issues.apache.org/jira/browse/YARN-5709) | Cleanup leader election configs and pluggability | Critical | resourcemanager | Karthik Kambatla | Karthik Kambatla | +| [HDFS-9483](https://issues.apache.org/jira/browse/HDFS-9483) | Documentation does not cover use of "swebhdfs" as URL scheme for SSL-secured WebHDFS. | Major | documentation | Chris Nauroth | Surendra Singh Lilhore | +| [HDFS-11292](https://issues.apache.org/jira/browse/HDFS-11292) | log lastWrittenTxId etc info in logSyncAll | Major | hdfs | Yongjun Zhang | Yongjun Zhang | +| [HADOOP-13956](https://issues.apache.org/jira/browse/HADOOP-13956) | Read ADLS credentials from Credential Provider | Critical | fs/adl | John Zhuge | John Zhuge | +| [HADOOP-13962](https://issues.apache.org/jira/browse/HADOOP-13962) | Update ADLS SDK to 2.1.4 | Major | fs/adl | John Zhuge | John Zhuge | +| [HADOOP-13037](https://issues.apache.org/jira/browse/HADOOP-13037) | Refactor Azure Data Lake Store as an independent FileSystem | Major | fs/adl | Shrikant Naidu | Vishwajeet Dusane | +| [HADOOP-13946](https://issues.apache.org/jira/browse/HADOOP-13946) | Document how HDFS updates timestamps in the FS spec; compare with object stores | Minor | documentation, fs | Steve Loughran | Steve Loughran | +| [HDFS-10601](https://issues.apache.org/jira/browse/HDFS-10601) | Improve log message to include hostname when the NameNode is in safemode | Minor | . | Kuhu Shukla | Kuhu Shukla | + + +### BUG FIXES: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HADOOP-8818](https://issues.apache.org/jira/browse/HADOOP-8818) | Should use equals() rather than == to compare String or Text in MD5MD5CRC32FileChecksum and TFileDumper | Minor | fs, io | Brandon Li | Brandon Li | +| [HADOOP-8436](https://issues.apache.org/jira/browse/HADOOP-8436) | NPE In getLocalPathForWrite ( path, conf ) when the required context item is not configured | Major | fs | Brahma Reddy Battula | Brahma Reddy Battula | +| [HADOOP-9121](https://issues.apache.org/jira/browse/HADOOP-9121) | InodeTree.java has redundant check for vName while throwing exception | Major | fs | Arup Malakar | Arup Malakar | +| [HADOOP-9242](https://issues.apache.org/jira/browse/HADOOP-9242) | Duplicate surefire plugin config in hadoop-common | Major | test | Andrey Klochkov | Andrey Klochkov | +| [HADOOP-8419](https://issues.apache.org/jira/browse/HADOOP-8419) | GzipCodec NPE upon reset with IBM JDK | Major | io | Luke Lu | Yu Li | +| [HDFS-4366](https://issues.apache.org/jira/browse/HDFS-4366) | Block Replication Policy Implementation May Skip Higher-Priority Blocks for Lower-Priority Blocks | Major | . | Derek Dagit | Derek Dagit | +| [HADOOP-11559](https://issues.apache.org/jira/browse/HADOOP-11559) | Add links to RackAwareness and InterfaceClassification to site index | Minor | documentation | Masatake Iwasaki | Masatake Iwasaki | +| [HADOOP-11581](https://issues.apache.org/jira/browse/HADOOP-11581) | Fix Multithreaded correctness Warnings #org.apache.hadoop.fs.shell.Ls | Major | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [HADOOP-11568](https://issues.apache.org/jira/browse/HADOOP-11568) | Description on usage of classpath in hadoop command is incomplete. | Trivial | tools | Archana T | Archana T | +| [HADOOP-10027](https://issues.apache.org/jira/browse/HADOOP-10027) | \*Compressor\_deflateBytesDirect passes instance instead of jclass to GetStaticObjectField | Minor | native | Eric Abbott | Hui Zheng | +| [HDFS-5356](https://issues.apache.org/jira/browse/HDFS-5356) | MiniDFSCluster shoud close all open FileSystems when shutdown() | Critical | test | haosdent | Rakesh R | +| [YARN-3197](https://issues.apache.org/jira/browse/YARN-3197) | Confusing log generated by CapacityScheduler | Minor | capacityscheduler | Hitesh Shah | Varun Saxena | +| [YARN-3243](https://issues.apache.org/jira/browse/YARN-3243) | CapacityScheduler should pass headroom from parent to children to make sure ParentQueue obey its capacity limits. | Major | capacityscheduler, resourcemanager | Wangda Tan | Wangda Tan | +| [YARN-3305](https://issues.apache.org/jira/browse/YARN-3305) | AM-Used Resource for leafqueue is wrongly populated if AM ResourceRequest is less than minimumAllocation | Major | scheduler | Rohith Sharma K S | Rohith Sharma K S | +| [YARN-3205](https://issues.apache.org/jira/browse/YARN-3205) | FileSystemRMStateStore should disable FileSystem Cache to avoid get a Filesystem with an old configuration. | Major | resourcemanager | zhihai xu | zhihai xu | +| [MAPREDUCE-5807](https://issues.apache.org/jira/browse/MAPREDUCE-5807) | Print usage for TeraSort job. | Trivial | examples | Rohith Sharma K S | Rohith Sharma K S | +| [YARN-3351](https://issues.apache.org/jira/browse/YARN-3351) | AppMaster tracking URL is broken in HA | Major | webapp | Anubhav Dhoot | Anubhav Dhoot | +| [HDFS-7867](https://issues.apache.org/jira/browse/HDFS-7867) | Update action param from "start" to "prepare" in rolling upgrade javadoc | Trivial | . | J.Andreina | J.Andreina | +| [MAPREDUCE-6281](https://issues.apache.org/jira/browse/MAPREDUCE-6281) | Fix javadoc in Terasort | Trivial | . | Albert Chu | Albert Chu | +| [YARN-3269](https://issues.apache.org/jira/browse/YARN-3269) | Yarn.nodemanager.remote-app-log-dir could not be configured to fully qualified path | Major | . | Xuan Gong | Xuan Gong | +| [MAPREDUCE-6213](https://issues.apache.org/jira/browse/MAPREDUCE-6213) | NullPointerException caused by job history server addr not resolvable | Minor | applicationmaster | Peng Zhang | Peng Zhang | +| [MAPREDUCE-5448](https://issues.apache.org/jira/browse/MAPREDUCE-5448) | MapFileOutputFormat#getReaders bug with invisible files/folders | Minor | mrv2 | Maysam Yabandeh | Maysam Yabandeh | +| [MAPREDUCE-6242](https://issues.apache.org/jira/browse/MAPREDUCE-6242) | Progress report log is incredibly excessive in application master | Major | applicationmaster | Jian Fang | Varun Saxena | +| [HDFS-3325](https://issues.apache.org/jira/browse/HDFS-3325) | When configuring "dfs.namenode.safemode.threshold-pct" to a value greater or equal to 1 there is mismatch in the UI report | Minor | . | J.Andreina | J.Andreina | +| [YARN-3383](https://issues.apache.org/jira/browse/YARN-3383) | AdminService should use "warn" instead of "info" to log exception when operation fails | Major | resourcemanager | Wangda Tan | Li Lu | +| [YARN-3397](https://issues.apache.org/jira/browse/YARN-3397) | yarn rmadmin should skip -failover | Minor | resourcemanager | J.Andreina | J.Andreina | +| [HADOOP-11724](https://issues.apache.org/jira/browse/HADOOP-11724) | DistCp throws NPE when the target directory is root. | Minor | . | Lei (Eddy) Xu | Lei (Eddy) Xu | +| [YARN-3400](https://issues.apache.org/jira/browse/YARN-3400) | [JDK 8] Build Failure due to unreported exceptions in RPCUtil | Major | . | Robert Kanter | Robert Kanter | +| [HDFS-7990](https://issues.apache.org/jira/browse/HDFS-7990) | IBR delete ack should not be delayed | Major | namenode | Daryn Sharp | Daryn Sharp | +| [HADOOP-11760](https://issues.apache.org/jira/browse/HADOOP-11760) | Fix typo of javadoc in DistCp | Trivial | . | Chen He | Brahma Reddy Battula | +| [MAPREDUCE-6294](https://issues.apache.org/jira/browse/MAPREDUCE-6294) | Remove an extra parameter described in Javadoc of TockenCache | Trivial | . | Chen He | Brahma Reddy Battula | +| [HDFS-7501](https://issues.apache.org/jira/browse/HDFS-7501) | TransactionsSinceLastCheckpoint can be negative on SBNs | Major | namenode | Harsh J | Gautam Gopalakrishnan | +| [HDFS-8002](https://issues.apache.org/jira/browse/HDFS-8002) | Website refers to /trash directory | Major | documentation | Mike Drob | Brahma Reddy Battula | +| [HDFS-7261](https://issues.apache.org/jira/browse/HDFS-7261) | storageMap is accessed without synchronization in DatanodeDescriptor#updateHeartbeatState() | Major | . | Ted Yu | Brahma Reddy Battula | +| [HDFS-7997](https://issues.apache.org/jira/browse/HDFS-7997) | The first non-existing xattr should also throw IOException | Minor | . | zhouyingchao | zhouyingchao | +| [HDFS-6945](https://issues.apache.org/jira/browse/HDFS-6945) | BlockManager should remove a block from excessReplicateMap and decrement ExcessBlocks metric when the block is removed | Critical | namenode | Akira Ajisaka | Akira Ajisaka | +| [YARN-3425](https://issues.apache.org/jira/browse/YARN-3425) | NPE from RMNodeLabelsManager.serviceStop when NodeLabelsManager.serviceInit failed | Minor | resourcemanager | Bibin A Chundatt | Bibin A Chundatt | +| [HDFS-7922](https://issues.apache.org/jira/browse/HDFS-7922) | ShortCircuitCache#close is not releasing ScheduledThreadPoolExecutors | Major | . | Rakesh R | Rakesh R | +| [HDFS-8026](https://issues.apache.org/jira/browse/HDFS-8026) | Trace FSOutputSummer#writeChecksumChunks rather than DFSOutputStream#writeChunk | Minor | . | Colin P. McCabe | Colin P. McCabe | +| [YARN-3415](https://issues.apache.org/jira/browse/YARN-3415) | Non-AM containers can be counted towards amResourceUsage of a Fair Scheduler queue | Critical | fairscheduler | Rohit Agarwal | zhihai xu | +| [HADOOP-11797](https://issues.apache.org/jira/browse/HADOOP-11797) | releasedocmaker.py needs to put ASF headers on output | Major | build | Allen Wittenauer | Allen Wittenauer | +| [HADOOP-11800](https://issues.apache.org/jira/browse/HADOOP-11800) | Clean up some test methods in TestCodec.java | Major | test | Akira Ajisaka | Brahma Reddy Battula | +| [MAPREDUCE-4844](https://issues.apache.org/jira/browse/MAPREDUCE-4844) | Counters / AbstractCounters have constant references not declared final | Major | . | Gera Shegalov | Brahma Reddy Battula | +| [YARN-3435](https://issues.apache.org/jira/browse/YARN-3435) | AM container to be allocated Appattempt AM container shown as null | Trivial | resourcemanager | Bibin A Chundatt | Bibin A Chundatt | +| [YARN-3429](https://issues.apache.org/jira/browse/YARN-3429) | TestAMRMTokens.testTokenExpiry fails Intermittently with error message:Invalid AMRMToken | Major | test | zhihai xu | zhihai xu | +| [HDFS-5215](https://issues.apache.org/jira/browse/HDFS-5215) | dfs.datanode.du.reserved is not considered while computing available space | Major | datanode | Brahma Reddy Battula | Brahma Reddy Battula | +| [YARN-3457](https://issues.apache.org/jira/browse/YARN-3457) | NPE when NodeManager.serviceInit fails and stopRecoveryStore called | Minor | nodemanager | Bibin A Chundatt | Bibin A Chundatt | +| [YARN-3459](https://issues.apache.org/jira/browse/YARN-3459) | Fix failiure of TestLog4jWarningErrorMetricsAppender | Blocker | . | Li Lu | Varun Vasudev | +| [HDFS-8046](https://issues.apache.org/jira/browse/HDFS-8046) | Allow better control of getContentSummary | Major | . | Kihwal Lee | Kihwal Lee | +| [YARN-2890](https://issues.apache.org/jira/browse/YARN-2890) | MiniYarnCluster should turn on timeline service if configured to do so | Major | . | Mit Desai | Mit Desai | +| [HDFS-7725](https://issues.apache.org/jira/browse/HDFS-7725) | Incorrect "nodes in service" metrics caused all writes to fail | Major | . | Ming Ma | Ming Ma | +| [HDFS-8096](https://issues.apache.org/jira/browse/HDFS-8096) | DatanodeMetrics#blocksReplicated will get incremented early and even for failed transfers | Major | datanode | Vinayakumar B | Vinayakumar B | +| [YARN-3465](https://issues.apache.org/jira/browse/YARN-3465) | Use LinkedHashMap to preserve order of resource requests | Major | nodemanager | zhihai xu | zhihai xu | +| [HDFS-8099](https://issues.apache.org/jira/browse/HDFS-8099) | Change "DFSInputStream has been closed already" message to debug log level | Minor | hdfs-client | Charles Lamb | Charles Lamb | +| [HDFS-8091](https://issues.apache.org/jira/browse/HDFS-8091) | ACLStatus and XAttributes not properly presented to INodeAttributesProvider before returning to client | Major | namenode | Arun Suresh | Arun Suresh | +| [MAPREDUCE-6266](https://issues.apache.org/jira/browse/MAPREDUCE-6266) | Job#getTrackingURL should consistently return a proper URL | Minor | . | Ray Chiang | Ray Chiang | +| [HDFS-8081](https://issues.apache.org/jira/browse/HDFS-8081) | Split getAdditionalBlock() into two methods. | Major | . | Konstantin Shvachko | Konstantin Shvachko | +| [HDFS-7939](https://issues.apache.org/jira/browse/HDFS-7939) | Two fsimage\_rollback\_\* files are created which are not deleted after rollback. | Critical | . | J.Andreina | J.Andreina | +| [MAPREDUCE-6314](https://issues.apache.org/jira/browse/MAPREDUCE-6314) | TestPipeApplication fails on trunk | Major | test | Varun Vasudev | Varun Vasudev | +| [YARN-3466](https://issues.apache.org/jira/browse/YARN-3466) | Fix RM nodes web page to sort by node HTTP-address, #containers and node-label column | Major | resourcemanager, webapp | Jason Lowe | Jason Lowe | +| [HDFS-7931](https://issues.apache.org/jira/browse/HDFS-7931) | DistributedFIleSystem should not look for keyProvider in cache if Encryption is disabled | Minor | hdfs-client | Arun Suresh | Arun Suresh | +| [HDFS-8111](https://issues.apache.org/jira/browse/HDFS-8111) | NPE thrown when invalid FSImage filename given for "hdfs oiv\_legacy" cmd | Minor | tools | Archana T | Surendra Singh Lilhore | +| [HADOOP-11811](https://issues.apache.org/jira/browse/HADOOP-11811) | Fix typos in hadoop-project/pom.xml and TestAccessControlList | Minor | . | Chen He | Brahma Reddy Battula | +| [YARN-3382](https://issues.apache.org/jira/browse/YARN-3382) | Some of UserMetricsInfo metrics are incorrectly set to root queue metrics | Major | webapp | Rohit Agarwal | Rohit Agarwal | +| [YARN-3472](https://issues.apache.org/jira/browse/YARN-3472) | Possible leak in DelegationTokenRenewer#allTokens | Major | . | Jian He | Rohith Sharma K S | +| [YARN-3394](https://issues.apache.org/jira/browse/YARN-3394) | WebApplication proxy documentation is incomplete | Minor | resourcemanager | Bibin A Chundatt | Naganarasimha G R | +| [HADOOP-11819](https://issues.apache.org/jira/browse/HADOOP-11819) | HttpServerFunctionalTest#prepareTestWebapp should create web app directory if it does not exist. | Minor | . | Rohith Sharma K S | Rohith Sharma K S | +| [HDFS-6666](https://issues.apache.org/jira/browse/HDFS-6666) | Abort NameNode and DataNode startup if security is enabled but block access token is not enabled. | Minor | datanode, namenode, security | Chris Nauroth | Vijay Bhat | +| [HDFS-8055](https://issues.apache.org/jira/browse/HDFS-8055) | NullPointerException when topology script is missing. | Major | namenode | Anu Engineer | Anu Engineer | +| [YARN-3266](https://issues.apache.org/jira/browse/YARN-3266) | RMContext inactiveNodes should have NodeId as map key | Major | resourcemanager | Chengbing Liu | Chengbing Liu | +| [YARN-3436](https://issues.apache.org/jira/browse/YARN-3436) | Fix URIs in documention of YARN web service REST APIs | Minor | documentation, resourcemanager | Bibin A Chundatt | Bibin A Chundatt | +| [HDFS-8127](https://issues.apache.org/jira/browse/HDFS-8127) | NameNode Failover during HA upgrade can cause DataNode to finalize upgrade | Blocker | ha | Jing Zhao | Jing Zhao | +| [YARN-3462](https://issues.apache.org/jira/browse/YARN-3462) | Patches applied for YARN-2424 are inconsistent between trunk and branch-2 | Major | . | Sidharta Seethana | Naganarasimha G R | +| [HDFS-8151](https://issues.apache.org/jira/browse/HDFS-8151) | Always use snapshot path as source when invalid snapshot names are used for diff based distcp | Minor | distcp | Sushmitha Sreenivasan | Jing Zhao | +| [HDFS-7934](https://issues.apache.org/jira/browse/HDFS-7934) | Update RollingUpgrade rollback documentation: should use bootstrapstandby for standby NN | Critical | documentation | J.Andreina | J.Andreina | +| [HDFS-8149](https://issues.apache.org/jira/browse/HDFS-8149) | The footer of the Web UI "Hadoop, 2014" is old | Major | . | Akira Ajisaka | Brahma Reddy Battula | +| [HDFS-8142](https://issues.apache.org/jira/browse/HDFS-8142) | DistributedFileSystem encryption zone commands should resolve relative paths | Major | . | Rakesh R | Rakesh R | +| [MAPREDUCE-6300](https://issues.apache.org/jira/browse/MAPREDUCE-6300) | Task list sort by task id broken | Minor | . | Siqi Li | Siqi Li | +| [YARN-3021](https://issues.apache.org/jira/browse/YARN-3021) | YARN's delegation-token handling disallows certain trust setups to operate properly over DistCp | Major | security | Harsh J | Yongjun Zhang | +| [HDFS-8153](https://issues.apache.org/jira/browse/HDFS-8153) | Error Message points to wrong parent directory in case of path component name length error | Major | namenode | Anu Engineer | Anu Engineer | +| [YARN-3493](https://issues.apache.org/jira/browse/YARN-3493) | RM fails to come up with error "Failed to load/recover state" when mem settings are changed | Critical | yarn | Sumana Sathish | Jian He | +| [HDFS-8043](https://issues.apache.org/jira/browse/HDFS-8043) | NPE in MiniDFSCluster teardown | Major | test | Steve Loughran | Brahma Reddy Battula | +| [HDFS-8173](https://issues.apache.org/jira/browse/HDFS-8173) | NPE thrown at DataNode shutdown when HTTP server was not able to create | Minor | datanode | Archana T | Surendra Singh Lilhore | +| [YARN-3497](https://issues.apache.org/jira/browse/YARN-3497) | ContainerManagementProtocolProxy modifies IPC timeout conf without making a copy | Major | client | Jason Lowe | Jason Lowe | +| [MAPREDUCE-6238](https://issues.apache.org/jira/browse/MAPREDUCE-6238) | MR2 can't run local jobs with -libjars command options which is a regression from MR1 | Critical | mrv2 | zhihai xu | zhihai xu | +| [HDFS-8179](https://issues.apache.org/jira/browse/HDFS-8179) | DFSClient#getServerDefaults returns null within 1 hour of system start | Blocker | . | Xiaoyu Yao | Xiaoyu Yao | +| [HDFS-7993](https://issues.apache.org/jira/browse/HDFS-7993) | Provide each Replica details in fsck | Major | . | Ming Ma | J.Andreina | +| [HDFS-8163](https://issues.apache.org/jira/browse/HDFS-8163) | Using monotonicNow for block report scheduling causes test failures on recently restarted systems | Blocker | datanode | Arpit Agarwal | Arpit Agarwal | +| [HADOOP-11704](https://issues.apache.org/jira/browse/HADOOP-11704) | DelegationTokenAuthenticationFilter must pass ipaddress instead of hostname to ProxyUsers#authorize() | Major | . | Anubhav Dhoot | Anubhav Dhoot | +| [YARN-3495](https://issues.apache.org/jira/browse/YARN-3495) | Confusing log generated by FairScheduler | Major | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [MAPREDUCE-6293](https://issues.apache.org/jira/browse/MAPREDUCE-6293) | Set job classloader on uber-job's LocalContainerLauncher event thread | Major | mr-am | Sangjin Lee | Sangjin Lee | +| [HADOOP-11846](https://issues.apache.org/jira/browse/HADOOP-11846) | TestCertificateUtil.testCorruptPEM failing on Jenkins JDK8 | Major | build, security | Steve Loughran | Larry McCay | +| [HADOOP-11859](https://issues.apache.org/jira/browse/HADOOP-11859) | PseudoAuthenticationHandler fails with httpcomponents v4.4 | Major | . | Eugene Koifman | Eugene Koifman | +| [MAPREDUCE-6330](https://issues.apache.org/jira/browse/MAPREDUCE-6330) | Fix typo in Task Attempt API's URL in documentations | Minor | documentation | Ryu Kobayashi | Ryu Kobayashi | +| [HADOOP-11848](https://issues.apache.org/jira/browse/HADOOP-11848) | Incorrect arguments to sizeof in DomainSocket.c | Major | native | Malcolm Kavalsky | Malcolm Kavalsky | +| [HADOOP-11868](https://issues.apache.org/jira/browse/HADOOP-11868) | Invalid user logins trigger large backtraces in server log | Major | . | Chang Li | Chang Li | +| [HADOOP-11861](https://issues.apache.org/jira/browse/HADOOP-11861) | test-patch.sh rewrite addendum patch | Major | build | Anu Engineer | Allen Wittenauer | +| [HADOOP-11864](https://issues.apache.org/jira/browse/HADOOP-11864) | JWTRedirectAuthenticationHandler breaks java8 javadocs | Major | build | Steve Loughran | Larry McCay | +| [HDFS-4448](https://issues.apache.org/jira/browse/HDFS-4448) | Allow HA NN to start in secure mode with wildcard address configured | Major | ha, namenode, security | Aaron T. Myers | Aaron T. Myers | +| [YARN-3522](https://issues.apache.org/jira/browse/YARN-3522) | DistributedShell uses the wrong user to put timeline data | Blocker | timelineserver | Zhijie Shen | Zhijie Shen | +| [HDFS-8147](https://issues.apache.org/jira/browse/HDFS-8147) | Mover should not schedule two replicas to the same DN storage | Major | balancer & mover | Surendra Singh Lilhore | Surendra Singh Lilhore | +| [HADOOP-11872](https://issues.apache.org/jira/browse/HADOOP-11872) | "hadoop dfs" command prints message about using "yarn jar" on Windows(branch-2 only) | Minor | scripts | Varun Vasudev | Varun Vasudev | +| [HADOOP-11730](https://issues.apache.org/jira/browse/HADOOP-11730) | Regression: s3n read failure recovery broken | Major | fs/s3 | Takenori Sato | Takenori Sato | +| [YARN-3516](https://issues.apache.org/jira/browse/YARN-3516) | killing ContainerLocalizer action doesn't take effect when private localizer receives FETCH\_FAILURE status. | Minor | nodemanager | zhihai xu | zhihai xu | +| [HADOOP-11802](https://issues.apache.org/jira/browse/HADOOP-11802) | DomainSocketWatcher thread terminates sometimes after there is an I/O error during requestShortCircuitShm | Major | . | Eric Payne | Colin P. McCabe | +| [HDFS-8070](https://issues.apache.org/jira/browse/HDFS-8070) | Pre-HDFS-7915 DFSClient cannot use short circuit on post-HDFS-7915 DataNode | Blocker | caching | Gopal V | Colin P. McCabe | +| [HDFS-8217](https://issues.apache.org/jira/browse/HDFS-8217) | During block recovery for truncate Log new Block Id in case of copy-on-truncate is true. | Major | datanode | Vinayakumar B | Vinayakumar B | +| [HDFS-8231](https://issues.apache.org/jira/browse/HDFS-8231) | StackTrace displayed at client while QuotaByStorageType exceeds | Major | hdfs-client | J.Andreina | J.Andreina | +| [HDFS-8191](https://issues.apache.org/jira/browse/HDFS-8191) | Fix byte to integer casting in SimulatedFSDataset#simulatedByte | Minor | . | Zhe Zhang | Zhe Zhang | +| [YARN-3387](https://issues.apache.org/jira/browse/YARN-3387) | Previous AM's container complete message couldn't pass to current am if am restarted and rm changed | Critical | resourcemanager | sandflee | sandflee | +| [HADOOP-11876](https://issues.apache.org/jira/browse/HADOOP-11876) | Refactor code to make it more readable, minor maybePrintStats bug | Trivial | tools/distcp | Zoran Dimitrijevic | Zoran Dimitrijevic | +| [YARN-3444](https://issues.apache.org/jira/browse/YARN-3444) | Fix typo capabililty | Trivial | applications/distributed-shell | Gabor Liptak | Gabor Liptak | +| [YARN-3537](https://issues.apache.org/jira/browse/YARN-3537) | NPE when NodeManager.serviceInit fails and stopRecoveryStore invoked | Major | nodemanager | Brahma Reddy Battula | Brahma Reddy Battula | +| [HDFS-8211](https://issues.apache.org/jira/browse/HDFS-8211) | DataNode UUID is always null in the JMX counter | Major | datanode | Anu Engineer | Anu Engineer | +| [MAPREDUCE-6333](https://issues.apache.org/jira/browse/MAPREDUCE-6333) | TestEvents,TestAMWebServicesTasks,TestAppController are broken due to MAPREDUCE-6297 | Major | . | Siqi Li | Siqi Li | +| [HDFS-8206](https://issues.apache.org/jira/browse/HDFS-8206) | Fix the typos in hadoop-hdfs-httpfs | Major | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [YARN-3464](https://issues.apache.org/jira/browse/YARN-3464) | Race condition in LocalizerRunner kills localizer before localizing all resources | Critical | nodemanager | zhihai xu | zhihai xu | +| [MAPREDUCE-6252](https://issues.apache.org/jira/browse/MAPREDUCE-6252) | JobHistoryServer should not fail when encountering a missing directory | Major | jobhistoryserver | Craig Welch | Craig Welch | +| [YARN-3530](https://issues.apache.org/jira/browse/YARN-3530) | ATS throws exception on trying to filter results without otherinfo. | Critical | timelineserver | Sreenath Somarajapuram | Zhijie Shen | +| [HDFS-8205](https://issues.apache.org/jira/browse/HDFS-8205) | CommandFormat#parse() should not parse option as value of option | Blocker | . | Peter Shi | Peter Shi | +| [HADOOP-11870](https://issues.apache.org/jira/browse/HADOOP-11870) | [JDK8] AuthenticationFilter, CertificateUtil, SignerSecretProviders, KeyAuthorizationKeyProvider Javadoc issues | Major | build | Robert Kanter | Robert Kanter | +| [MAPREDUCE-6324](https://issues.apache.org/jira/browse/MAPREDUCE-6324) | Uber jobs fail to update AMRM token when it rolls over | Blocker | mr-am | Jason Lowe | Jason Lowe | +| [HDFS-8232](https://issues.apache.org/jira/browse/HDFS-8232) | Missing datanode counters when using Metrics2 sink interface | Major | datanode | Anu Engineer | Anu Engineer | +| [MAPREDUCE-6334](https://issues.apache.org/jira/browse/MAPREDUCE-6334) | Fetcher#copyMapOutput is leaking usedMemory upon IOException during InMemoryMapOutput shuffle handler | Blocker | . | Eric Payne | Eric Payne | +| [HDFS-8273](https://issues.apache.org/jira/browse/HDFS-8273) | FSNamesystem#Delete() should not call logSync() when holding the lock | Blocker | namenode | Jing Zhao | Haohui Mai | +| [YARN-3485](https://issues.apache.org/jira/browse/YARN-3485) | FairScheduler headroom calculation doesn't consider maxResources for Fifo and FairShare policies | Critical | fairscheduler | Karthik Kambatla | Karthik Kambatla | +| [HDFS-8269](https://issues.apache.org/jira/browse/HDFS-8269) | getBlockLocations() does not resolve the .reserved path and generates incorrect edit logs when updating the atime | Blocker | . | Yesha Vora | Haohui Mai | +| [YARN-3517](https://issues.apache.org/jira/browse/YARN-3517) | RM web ui for dumping scheduler logs should be for admins only | Blocker | resourcemanager, security | Varun Vasudev | Varun Vasudev | +| [HDFS-8214](https://issues.apache.org/jira/browse/HDFS-8214) | Secondary NN Web UI shows wrong date for Last Checkpoint | Major | namenode | Charles Lamb | Charles Lamb | +| [YARN-3533](https://issues.apache.org/jira/browse/YARN-3533) | Test: Fix launchAM in MockRM to wait for attempt to be scheduled | Major | yarn | Anubhav Dhoot | Anubhav Dhoot | +| [MAPREDUCE-6339](https://issues.apache.org/jira/browse/MAPREDUCE-6339) | Job history file is not flushed correctly because isTimerActive flag is not set true when flushTimerTask is scheduled. | Critical | mrv2 | zhihai xu | zhihai xu | +| [HADOOP-11821](https://issues.apache.org/jira/browse/HADOOP-11821) | Fix findbugs warnings in hadoop-sls | Major | tools | Akira Ajisaka | Brahma Reddy Battula | +| [YARN-3564](https://issues.apache.org/jira/browse/YARN-3564) | Fix TestContainerAllocation.testAMContainerAllocationWhenDNSUnavailable fails randomly | Major | . | Jian He | Jian He | +| [HADOOP-11891](https://issues.apache.org/jira/browse/HADOOP-11891) | OsSecureRandom should lazily fill its reservoir | Major | security | Arun Suresh | Arun Suresh | +| [HADOOP-11866](https://issues.apache.org/jira/browse/HADOOP-11866) | increase readability and reliability of checkstyle, shellcheck, and whitespace reports | Minor | . | Naganarasimha G R | Allen Wittenauer | +| [HDFS-8292](https://issues.apache.org/jira/browse/HDFS-8292) | Move conditional in fmt\_time from dfs-dust.js to status.html | Minor | namenode | Charles Lamb | Charles Lamb | +| [HDFS-8300](https://issues.apache.org/jira/browse/HDFS-8300) | Fix unit test failures and findbugs warning caused by HDFS-8283 | Major | . | Jing Zhao | Jing Zhao | +| [HDFS-8276](https://issues.apache.org/jira/browse/HDFS-8276) | LazyPersistFileScrubber should be disabled if scrubber interval configured zero | Major | namenode | Surendra Singh Lilhore | Surendra Singh Lilhore | +| [HADOOP-11889](https://issues.apache.org/jira/browse/HADOOP-11889) | Make checkstyle runnable from root project | Major | build, test | Gera Shegalov | Gera Shegalov | +| [HDFS-8213](https://issues.apache.org/jira/browse/HDFS-8213) | DFSClient should use hdfs.client.htrace HTrace configuration prefix rather than hadoop.htrace | Critical | . | Billie Rinaldi | Colin P. McCabe | +| [HDFS-8229](https://issues.apache.org/jira/browse/HDFS-8229) | LAZY\_PERSIST file gets deleted after NameNode restart. | Major | namenode | Surendra Singh Lilhore | Surendra Singh Lilhore | +| [YARN-2893](https://issues.apache.org/jira/browse/YARN-2893) | AMLaucher: sporadic job failures due to EOFException in readTokenStorageStream | Major | resourcemanager | Gera Shegalov | zhihai xu | +| [HADOOP-11491](https://issues.apache.org/jira/browse/HADOOP-11491) | HarFs incorrectly declared as requiring an authority | Critical | fs | Gera Shegalov | Brahma Reddy Battula | +| [HADOOP-11900](https://issues.apache.org/jira/browse/HADOOP-11900) | Add failIfNoTests=false to hadoop-build-tools pom | Major | test | Gera Shegalov | Gera Shegalov | +| [MAPREDUCE-6345](https://issues.apache.org/jira/browse/MAPREDUCE-6345) | Documentation fix for when CRLA is enabled for MRAppMaster logs | Trivial | documentation | Rohit Agarwal | Rohit Agarwal | +| [YARN-2454](https://issues.apache.org/jira/browse/YARN-2454) | Fix compareTo of variable UNBOUNDED in o.a.h.y.util.resource.Resources. | Major | . | Xu Yang | Xu Yang | +| [YARN-1993](https://issues.apache.org/jira/browse/YARN-1993) | Cross-site scripting vulnerability in TextView.java | Major | webapp | Ted Yu | Kenji Kikushima | +| [MAPREDUCE-5905](https://issues.apache.org/jira/browse/MAPREDUCE-5905) | CountersStrings.toEscapedCompactStrings outputs unnecessary "null" strings | Minor | . | Akira Ajisaka | Akira Ajisaka | +| [MAPREDUCE-6349](https://issues.apache.org/jira/browse/MAPREDUCE-6349) | Fix typo in property org.apache.hadoop.mapreduce.lib.chain.Chain.REDUCER\_INPUT\_VALUE\_CLASS | Minor | . | Ray Chiang | Ray Chiang | +| [HADOOP-9658](https://issues.apache.org/jira/browse/HADOOP-9658) | SnappyCodec#checkNativeCodeLoaded may unexpectedly fail when native code is not loaded | Major | . | Zhijie Shen | Zhijie Shen | +| [YARN-3097](https://issues.apache.org/jira/browse/YARN-3097) | Logging of resource recovery on NM restart has redundancies | Minor | nodemanager | Jason Lowe | Eric Payne | +| [HDFS-8290](https://issues.apache.org/jira/browse/HDFS-8290) | WebHDFS calls before namesystem initialization can cause NullPointerException. | Minor | webhdfs | Chris Nauroth | Chris Nauroth | +| [MAPREDUCE-5649](https://issues.apache.org/jira/browse/MAPREDUCE-5649) | Reduce cannot use more than 2G memory for the final merge | Major | mrv2 | stanley shi | Gera Shegalov | +| [MAPREDUCE-6259](https://issues.apache.org/jira/browse/MAPREDUCE-6259) | IllegalArgumentException due to missing job submit time | Major | jobhistoryserver | zhihai xu | zhihai xu | +| [YARN-3375](https://issues.apache.org/jira/browse/YARN-3375) | NodeHealthScriptRunner.shouldRun() check is performing 3 times for starting NodeHealthScriptRunner | Minor | nodemanager | Devaraj K | Devaraj K | +| [YARN-2725](https://issues.apache.org/jira/browse/YARN-2725) | Adding test cases of retrying requests about ZKRMStateStore | Major | . | Tsuyoshi Ozawa | Tsuyoshi Ozawa | +| [MAPREDUCE-6165](https://issues.apache.org/jira/browse/MAPREDUCE-6165) | [JDK8] TestCombineFileInputFormat failed on JDK8 | Minor | . | Wei Yan | Akira Ajisaka | +| [HADOOP-11328](https://issues.apache.org/jira/browse/HADOOP-11328) | ZKFailoverController does not log Exception when doRun raises errors | Major | ha | Tianyin Xu | Tianyin Xu | +| [HADOOP-11916](https://issues.apache.org/jira/browse/HADOOP-11916) | TestStringUtils#testLowerAndUpperStrings failed on MAC due to a JVM bug | Minor | . | Ming Ma | Ming Ma | +| [YARN-3552](https://issues.apache.org/jira/browse/YARN-3552) | RM Web UI shows -1 running containers for completed apps | Trivial | webapp | Rohith Sharma K S | Rohith Sharma K S | +| [HADOOP-11120](https://issues.apache.org/jira/browse/HADOOP-11120) | hadoop fs -rmr gives wrong advice | Major | . | Allen Wittenauer | Juliet Hougland | +| [YARN-3396](https://issues.apache.org/jira/browse/YARN-3396) | Handle URISyntaxException in ResourceLocalizationService | Major | nodemanager | Chengbing Liu | Brahma Reddy Battula | +| [YARN-2123](https://issues.apache.org/jira/browse/YARN-2123) | Progress bars in Web UI always at 100% due to non-US locale | Major | webapp | Johannes Simon | Akira Ajisaka | +| [HDFS-8305](https://issues.apache.org/jira/browse/HDFS-8305) | HDFS INotify: the destination field of RenameOp should always end with the file name | Major | . | Colin P. McCabe | Colin P. McCabe | +| [HADOOP-11917](https://issues.apache.org/jira/browse/HADOOP-11917) | test-patch.sh should work with ${BASEDIR}/patchprocess setups | Blocker | . | Allen Wittenauer | Allen Wittenauer | +| [HDFS-7847](https://issues.apache.org/jira/browse/HDFS-7847) | Modify NNThroughputBenchmark to be able to operate on a remote NameNode | Major | . | Colin P. McCabe | Charles Lamb | +| [HDFS-8219](https://issues.apache.org/jira/browse/HDFS-8219) | setStoragePolicy with folder behavior is different after cluster restart | Major | . | Peter Shi | Surendra Singh Lilhore | +| [HADOOP-11898](https://issues.apache.org/jira/browse/HADOOP-11898) | add nfs3 and portmap starting command in hadoop-daemon.sh in branch-2 | Minor | bin, nfs | Brandon Li | Brandon Li | +| [HADOOP-11926](https://issues.apache.org/jira/browse/HADOOP-11926) | test-patch.sh mv does wrong math | Major | . | Allen Wittenauer | Allen Wittenauer | +| [HADOOP-11912](https://issues.apache.org/jira/browse/HADOOP-11912) | Extra configuration key used in TraceUtils should respect prefix | Minor | . | Masatake Iwasaki | Masatake Iwasaki | +| [YARN-3582](https://issues.apache.org/jira/browse/YARN-3582) | NPE in WebAppProxyServlet | Major | . | Jian He | Jian He | +| [HDFS-2484](https://issues.apache.org/jira/browse/HDFS-2484) | checkLease should throw FileNotFoundException when file does not exist | Major | namenode | Konstantin Shvachko | Rakesh R | +| [YARN-3385](https://issues.apache.org/jira/browse/YARN-3385) | Race condition: KeeperException$NoNodeException will cause RM shutdown during ZK node deletion. | Critical | resourcemanager | zhihai xu | zhihai xu | +| [HDFS-7833](https://issues.apache.org/jira/browse/HDFS-7833) | DataNode reconfiguration does not recalculate valid volumes required, based on configured failed volumes tolerated. | Major | datanode | Chris Nauroth | Lei (Eddy) Xu | +| [YARN-3577](https://issues.apache.org/jira/browse/YARN-3577) | Misspelling of threshold in log4j.properties for tests | Minor | test | Brahma Reddy Battula | Brahma Reddy Battula | +| [HDFS-8325](https://issues.apache.org/jira/browse/HDFS-8325) | Misspelling of threshold in log4j.properties for tests | Minor | test | Brahma Reddy Battula | Brahma Reddy Battula | +| [HADOOP-10387](https://issues.apache.org/jira/browse/HADOOP-10387) | Misspelling of threshold in log4j.properties for tests in hadoop-common-project | Minor | conf, test | Kenji Kikushima | Brahma Reddy Battula | +| [MAPREDUCE-6356](https://issues.apache.org/jira/browse/MAPREDUCE-6356) | Misspelling of threshold in log4j.properties for tests | Minor | test | Brahma Reddy Battula | Brahma Reddy Battula | +| [YARN-3523](https://issues.apache.org/jira/browse/YARN-3523) | Cleanup ResourceManagerAdministrationProtocol interface audience | Major | client, resourcemanager | Wangda Tan | Naganarasimha G R | +| [HDFS-7980](https://issues.apache.org/jira/browse/HDFS-7980) | Incremental BlockReport will dramatically slow down the startup of a namenode | Major | . | Hui Zheng | Walter Su | +| [HADOOP-11936](https://issues.apache.org/jira/browse/HADOOP-11936) | Dockerfile references a removed image | Major | . | Allen Wittenauer | Allen Wittenauer | +| [YARN-3584](https://issues.apache.org/jira/browse/YARN-3584) | [Log mesage correction] : MIssing space in Diagnostics message | Trivial | . | nijel | nijel | +| [HDFS-8321](https://issues.apache.org/jira/browse/HDFS-8321) | CacheDirectives and CachePool operations should throw RetriableException in safemode | Major | . | Haohui Mai | Haohui Mai | +| [HDFS-8037](https://issues.apache.org/jira/browse/HDFS-8037) | CheckAccess in WebHDFS silently accepts malformed FsActions parameters | Minor | webhdfs | Jake Low | Walter Su | +| [YARN-1832](https://issues.apache.org/jira/browse/YARN-1832) | Fix wrong MockLocalizerStatus#equals implementation | Minor | nodemanager | Hong Zhiguo | Hong Zhiguo | +| [YARN-3572](https://issues.apache.org/jira/browse/YARN-3572) | Correct typos in WritingYarnApplications.md | Minor | documentation | Sandeep Khurana | Gabor Liptak | +| [HADOOP-11922](https://issues.apache.org/jira/browse/HADOOP-11922) | Misspelling of threshold in log4j.properties for tests in hadoop-tools | Minor | . | Brahma Reddy Battula | Gabor Liptak | +| [HDFS-8257](https://issues.apache.org/jira/browse/HDFS-8257) | Namenode rollingUpgrade option is incorrect in document | Major | documentation | J.Andreina | J.Andreina | +| [HDFS-8067](https://issues.apache.org/jira/browse/HDFS-8067) | haadmin prints out stale help messages | Minor | hdfs-client | Ajith S | Ajith S | +| [YARN-3592](https://issues.apache.org/jira/browse/YARN-3592) | Fix typos in RMNodeLabelsManager | Trivial | resourcemanager | Junping Du | Sunil G | +| [HDFS-8174](https://issues.apache.org/jira/browse/HDFS-8174) | Update replication count to live rep count in fsck report | Minor | . | J.Andreina | J.Andreina | +| [HDFS-6291](https://issues.apache.org/jira/browse/HDFS-6291) | FSImage may be left unclosed in BootstrapStandby#doRun() | Minor | ha | Ted Yu | Sanghyun Yun | +| [YARN-3358](https://issues.apache.org/jira/browse/YARN-3358) | Audit log not present while refreshing Service ACLs | Minor | resourcemanager | Varun Saxena | Varun Saxena | +| [YARN-3589](https://issues.apache.org/jira/browse/YARN-3589) | RM and AH web UI display DOCTYPE wrongly | Major | webapp | Rohith Sharma K S | Rohith Sharma K S | +| [HDFS-7998](https://issues.apache.org/jira/browse/HDFS-7998) | HDFS Federation : Command mentioned to add a NN to existing federated cluster is wrong | Minor | documentation | Ajith S | Ajith S | +| [HDFS-8222](https://issues.apache.org/jira/browse/HDFS-8222) | Remove usage of "dfsadmin -upgradeProgress " from document which is no longer supported | Major | documentation | J.Andreina | J.Andreina | +| [HDFS-8187](https://issues.apache.org/jira/browse/HDFS-8187) | Remove usage of "-setStoragePolicy" and "-getStoragePolicy" using dfsadmin cmd (as it is not been supported) | Major | documentation | J.Andreina | J.Andreina | +| [HDFS-8175](https://issues.apache.org/jira/browse/HDFS-8175) | Provide information on snapshotDiff for supporting the comparison between snapshot and current status | Major | documentation | J.Andreina | J.Andreina | +| [MAPREDUCE-6342](https://issues.apache.org/jira/browse/MAPREDUCE-6342) | Make POM project names consistent | Minor | build | Rohith Sharma K S | Rohith Sharma K S | +| [HDFS-6576](https://issues.apache.org/jira/browse/HDFS-6576) | Datanode log is generating at root directory in security mode | Minor | datanode, scripts | Surendra Singh Lilhore | Surendra Singh Lilhore | +| [HADOOP-11877](https://issues.apache.org/jira/browse/HADOOP-11877) | SnappyDecompressor's Logger class name is wrong | Major | conf | Surendra Singh Lilhore | Surendra Singh Lilhore | +| [HDFS-3384](https://issues.apache.org/jira/browse/HDFS-3384) | DataStreamer thread should be closed immediatly when failed to setup a PipelineForAppendOrRecovery | Major | hdfs-client | Brahma Reddy Battula | Uma Maheswara Rao G | +| [YARN-3554](https://issues.apache.org/jira/browse/YARN-3554) | Default value for maximum nodemanager connect wait time is too high | Major | . | Jason Lowe | Naganarasimha G R | +| [HDFS-7894](https://issues.apache.org/jira/browse/HDFS-7894) | Rolling upgrade readiness is not updated in jmx until query command is issued. | Critical | . | Kihwal Lee | Brahma Reddy Battula | +| [YARN-3600](https://issues.apache.org/jira/browse/YARN-3600) | AM container link is broken (on a killed application, at least) | Major | . | Sergey Shelukhin | Naganarasimha G R | +| [HDFS-8346](https://issues.apache.org/jira/browse/HDFS-8346) | libwebhdfs build fails during link due to unresolved external symbols. | Major | native | Chris Nauroth | Chris Nauroth | +| [HDFS-8274](https://issues.apache.org/jira/browse/HDFS-8274) | NFS configuration nfs.dump.dir not working | Major | nfs | Ajith S | Ajith S | +| [HDFS-8340](https://issues.apache.org/jira/browse/HDFS-8340) | Fix NFS documentation of nfs.wtmax | Minor | documentation, nfs | Ajith S | Ajith S | +| [HADOOP-10356](https://issues.apache.org/jira/browse/HADOOP-10356) | Corrections in winutils/chmod.c | Trivial | bin | René Nyffenegger | René Nyffenegger | +| [HADOOP-7165](https://issues.apache.org/jira/browse/HADOOP-7165) | listLocatedStatus(path, filter) is not redefined in FilterFs | Major | fs | Hairong Kuang | Hairong Kuang | +| [MAPREDUCE-3383](https://issues.apache.org/jira/browse/MAPREDUCE-3383) | Duplicate job.getOutputValueGroupingComparator() in ReduceTask | Major | . | Binglin Chang | Binglin Chang | +| [HDFS-8311](https://issues.apache.org/jira/browse/HDFS-8311) | DataStreamer.transfer() should timeout the socket InputStream. | Major | hdfs-client | Esteban Gutierrez | Esteban Gutierrez | +| [HDFS-8113](https://issues.apache.org/jira/browse/HDFS-8113) | Add check for null BlockCollection pointers in BlockInfoContiguous structures | Major | namenode | Chengbing Liu | Chengbing Liu | +| [HADOOP-9729](https://issues.apache.org/jira/browse/HADOOP-9729) | The example code of org.apache.hadoop.util.Tool is incorrect | Major | util | hellojinjie | hellojinjie | +| [MAPREDUCE-2094](https://issues.apache.org/jira/browse/MAPREDUCE-2094) | LineRecordReader should not seek into non-splittable, compressed streams. | Major | task | Niels Basjes | Niels Basjes | +| [HDFS-8245](https://issues.apache.org/jira/browse/HDFS-8245) | Standby namenode doesn't process DELETED\_BLOCK if the add block request is in edit log. | Major | . | Rushabh S Shah | Rushabh S Shah | +| [YARN-3018](https://issues.apache.org/jira/browse/YARN-3018) | Unify the default value for yarn.scheduler.capacity.node-locality-delay in code and default xml file | Trivial | capacityscheduler | nijel | nijel | +| [HDFS-8326](https://issues.apache.org/jira/browse/HDFS-8326) | Documentation about when checkpoints are run is out of date | Major | documentation | Misty Stanley-Jones | Misty Stanley-Jones | +| [YARN-3604](https://issues.apache.org/jira/browse/YARN-3604) | removeApplication in ZKRMStateStore should also disable watch. | Minor | resourcemanager | zhihai xu | zhihai xu | +| [YARN-3476](https://issues.apache.org/jira/browse/YARN-3476) | Nodemanager can fail to delete local logs if log aggregation fails | Major | log-aggregation, nodemanager | Jason Lowe | Rohith Sharma K S | +| [YARN-3473](https://issues.apache.org/jira/browse/YARN-3473) | Fix RM Web UI configuration for some properties | Minor | resourcemanager | Ray Chiang | Ray Chiang | +| [HDFS-8097](https://issues.apache.org/jira/browse/HDFS-8097) | TestFileTruncate is failing intermittently | Major | test | Rakesh R | Rakesh R | +| [YARN-1912](https://issues.apache.org/jira/browse/YARN-1912) | ResourceLocalizer started without any jvm memory control | Major | nodemanager | stanley shi | Masatake Iwasaki | +| [MAPREDUCE-6359](https://issues.apache.org/jira/browse/MAPREDUCE-6359) | RM HA setup, "Cluster" tab links populated with AM hostname instead of RM | Minor | . | Aroop Maliakkal | yunjiong zhao | +| [MAPREDUCE-6353](https://issues.apache.org/jira/browse/MAPREDUCE-6353) | Divide by zero error in MR AM when calculating available containers | Major | mr-am | Anubhav Dhoot | Anubhav Dhoot | +| [YARN-3395](https://issues.apache.org/jira/browse/YARN-3395) | FairScheduler: Trim whitespaces when using username for queuename | Major | fairscheduler | zhihai xu | zhihai xu | +| [HDFS-8351](https://issues.apache.org/jira/browse/HDFS-8351) | Remove namenode -finalize option from document | Major | documentation | Akira Ajisaka | Akira Ajisaka | +| [YARN-3587](https://issues.apache.org/jira/browse/YARN-3587) | Fix the javadoc of DelegationTokenSecretManager in projects of yarn, etc. | Minor | documentation | Akira Ajisaka | Gabor Liptak | +| [HADOOP-11663](https://issues.apache.org/jira/browse/HADOOP-11663) | Remove description about Java 6 from docs | Minor | documentation | Masatake Iwasaki | Masatake Iwasaki | +| [HADOOP-11928](https://issues.apache.org/jira/browse/HADOOP-11928) | Test-patch check for @author tags incorrectly flags removal of @author tags | Major | . | Sean Busbey | Kengo Seki | +| [HADOOP-11951](https://issues.apache.org/jira/browse/HADOOP-11951) | test-patch should give better info about failures to handle dev-support updates without resetrepo option | Minor | . | Sean Busbey | Sean Busbey | +| [HADOOP-11947](https://issues.apache.org/jira/browse/HADOOP-11947) | test-patch should return early from determine-issue when run in jenkins mode. | Minor | . | Sean Busbey | Sean Busbey | +| [HDFS-7916](https://issues.apache.org/jira/browse/HDFS-7916) | 'reportBadBlocks' from datanodes to standby Node BPServiceActor goes for infinite loop | Critical | datanode | Vinayakumar B | Rushabh S Shah | +| [YARN-3434](https://issues.apache.org/jira/browse/YARN-3434) | Interaction between reservations and userlimit can result in significant ULF violation | Major | capacityscheduler | Thomas Graves | Thomas Graves | +| [HDFS-8362](https://issues.apache.org/jira/browse/HDFS-8362) | Java Compilation Error in TestHdfsConfigFields.java | Major | . | Mohammad Arshad | Mohammad Arshad | +| [MAPREDUCE-6360](https://issues.apache.org/jira/browse/MAPREDUCE-6360) | TestMapreduceConfigFields is placed in wrong dir, introducing compile error | Major | . | Vinayakumar B | Mohammad Arshad | +| [MAPREDUCE-6361](https://issues.apache.org/jira/browse/MAPREDUCE-6361) | NPE issue in shuffle caused by concurrent issue between copySucceeded() in one thread and copyFailed() in another thread on the same host | Critical | . | Junping Du | Junping Du | +| [YARN-3629](https://issues.apache.org/jira/browse/YARN-3629) | NodeID is always printed as "null" in node manager initialization log. | Major | . | nijel | nijel | +| [MAPREDUCE-6251](https://issues.apache.org/jira/browse/MAPREDUCE-6251) | JobClient needs additional retries at a higher level to address not-immediately-consistent dfs corner cases | Major | jobhistoryserver, mrv2 | Craig Welch | Craig Welch | +| [MAPREDUCE-6366](https://issues.apache.org/jira/browse/MAPREDUCE-6366) | mapreduce.terasort.final.sync configuration in TeraSort doesn't work | Trivial | examples | Takuya Fukudome | Takuya Fukudome | +| [HDFS-6300](https://issues.apache.org/jira/browse/HDFS-6300) | Prevent multiple balancers from running simultaneously | Critical | balancer & mover | Rakesh R | Rakesh R | +| [HDFS-8358](https://issues.apache.org/jira/browse/HDFS-8358) | TestTraceAdmin fails | Major | . | Kihwal Lee | Masatake Iwasaki | +| [HADOOP-11966](https://issues.apache.org/jira/browse/HADOOP-11966) | Variable cygwin is undefined in hadoop-config.sh when executed through hadoop-daemon.sh. | Critical | scripts | Chris Nauroth | Chris Nauroth | +| [HDFS-8380](https://issues.apache.org/jira/browse/HDFS-8380) | Always call addStoredBlock on blocks which have been shifted from one storage to another | Major | . | Colin P. McCabe | Colin P. McCabe | +| [HADOOP-8174](https://issues.apache.org/jira/browse/HADOOP-8174) | Remove confusing comment in Path#isAbsolute() | Trivial | fs | Suresh Srinivas | Suresh Srinivas | +| [HDFS-8150](https://issues.apache.org/jira/browse/HDFS-8150) | Make getFileChecksum fail for blocks under construction | Critical | . | Kihwal Lee | J.Andreina | +| [MAPREDUCE-5708](https://issues.apache.org/jira/browse/MAPREDUCE-5708) | Duplicate String.format in YarnOutputFiles.getSpillFileForWrite | Minor | . | Konstantin Weitz | Konstantin Weitz | +| [YARN-1519](https://issues.apache.org/jira/browse/YARN-1519) | check if sysconf is implemented before using it | Major | nodemanager | Radim Kolar | Radim Kolar | +| [HDFS-8371](https://issues.apache.org/jira/browse/HDFS-8371) | Fix test failure in TestHdfsConfigFields for spanreceiver properties | Major | . | Ray Chiang | Ray Chiang | +| [MAPREDUCE-6273](https://issues.apache.org/jira/browse/MAPREDUCE-6273) | HistoryFileManager should check whether summaryFile exists to avoid FileNotFoundException causing HistoryFileInfo into MOVE\_FAILED state | Minor | jobhistoryserver | zhihai xu | zhihai xu | +| [YARN-2421](https://issues.apache.org/jira/browse/YARN-2421) | RM still allocates containers to an app in the FINISHING state | Major | scheduler | Thomas Graves | Chang Li | +| [YARN-3526](https://issues.apache.org/jira/browse/YARN-3526) | ApplicationMaster tracking URL is incorrectly redirected on a QJM cluster | Major | resourcemanager, webapp | Weiwei Yang | Weiwei Yang | +| [HADOOP-11988](https://issues.apache.org/jira/browse/HADOOP-11988) | Fix typo in the document for hadoop fs -find | Trivial | documentation | Akira Ajisaka | Kengo Seki | +| [HADOOP-10582](https://issues.apache.org/jira/browse/HADOOP-10582) | Fix the test case for copying to non-existent dir in TestFsShellCopy | Minor | fs | Kousuke Saruta | Kousuke Saruta | +| [HDFS-8345](https://issues.apache.org/jira/browse/HDFS-8345) | Storage policy APIs must be exposed via the FileSystem interface | Major | hdfs-client | Arpit Agarwal | Arpit Agarwal | +| [HDFS-8405](https://issues.apache.org/jira/browse/HDFS-8405) | Fix a typo in NamenodeFsck | Minor | namenode | Tsz Wo Nicholas Sze | Takanobu Asanuma | +| [HDFS-6348](https://issues.apache.org/jira/browse/HDFS-6348) | SecondaryNameNode not terminating properly on runtime exceptions | Major | namenode | Rakesh R | Rakesh R | +| [YARN-3601](https://issues.apache.org/jira/browse/YARN-3601) | Fix UT TestRMFailover.testRMWebAppRedirect | Critical | resourcemanager, webapp | Weiwei Yang | Weiwei Yang | +| [HDFS-8404](https://issues.apache.org/jira/browse/HDFS-8404) | Pending block replication can get stuck using older genstamp | Major | namenode | Nathan Roberts | Nathan Roberts | +| [HADOOP-11973](https://issues.apache.org/jira/browse/HADOOP-11973) | Ensure ZkDelegationTokenSecretManager namespace znodes get created with ACLs | Major | security | Gregory Chanan | Gregory Chanan | +| [HADOOP-11963](https://issues.apache.org/jira/browse/HADOOP-11963) | Metrics documentation for FSNamesystem misspells PendingDataNodeMessageCount. | Trivial | documentation | Chris Nauroth | Anu Engineer | +| [YARN-2821](https://issues.apache.org/jira/browse/YARN-2821) | Distributed shell app master becomes unresponsive sometimes | Major | applications/distributed-shell | Varun Vasudev | Varun Vasudev | +| [YARN-3677](https://issues.apache.org/jira/browse/YARN-3677) | Fix findbugs warnings in yarn-server-resourcemanager | Minor | resourcemanager | Akira Ajisaka | Vinod Kumar Vavilapalli | +| [YARN-3681](https://issues.apache.org/jira/browse/YARN-3681) | yarn cmd says "could not find main class 'queue'" in windows | Blocker | yarn | Sumana Sathish | Varun Saxena | +| [YARN-3654](https://issues.apache.org/jira/browse/YARN-3654) | ContainerLogsPage web UI should not have meta-refresh | Major | yarn | Xuan Gong | Xuan Gong | +| [YARN-3694](https://issues.apache.org/jira/browse/YARN-3694) | Fix dead link for TimelineServer REST API | Minor | documentation | Akira Ajisaka | Jagadesh Kiran N | +| [YARN-3646](https://issues.apache.org/jira/browse/YARN-3646) | Applications are getting stuck some times in case of retry policy forever | Major | client | Raju Bairishetti | Raju Bairishetti | +| [HDFS-8421](https://issues.apache.org/jira/browse/HDFS-8421) | Move startFile() and related operations into FSDirWriteFileOp | Major | . | Haohui Mai | Haohui Mai | +| [HDFS-8451](https://issues.apache.org/jira/browse/HDFS-8451) | DFSClient probe for encryption testing interprets empty URI property for "enabled" | Blocker | encryption | Steve Loughran | Steve Loughran | +| [YARN-3675](https://issues.apache.org/jira/browse/YARN-3675) | FairScheduler: RM quits when node removal races with continousscheduling on the same node | Critical | fairscheduler | Anubhav Dhoot | Anubhav Dhoot | +| [HADOOP-12014](https://issues.apache.org/jira/browse/HADOOP-12014) | hadoop-config.cmd displays a wrong error message | Minor | scripts | Kengo Seki | Kengo Seki | +| [HADOOP-11955](https://issues.apache.org/jira/browse/HADOOP-11955) | Fix a typo in the cluster setup doc | Trivial | . | Kihwal Lee | Yanjun Wang | +| [HDFS-8268](https://issues.apache.org/jira/browse/HDFS-8268) | Port conflict log for data node server is not sufficient | Minor | datanode | Mohammad Shahid Khan | Mohammad Shahid Khan | +| [YARN-3594](https://issues.apache.org/jira/browse/YARN-3594) | WintuilsProcessStubExecutor.startStreamReader leaks streams | Trivial | nodemanager | Steve Loughran | Lars Francke | +| [HADOOP-11743](https://issues.apache.org/jira/browse/HADOOP-11743) | maven doesn't clean all the site files | Minor | documentation | Allen Wittenauer | ramtin | +| [HADOOP-11927](https://issues.apache.org/jira/browse/HADOOP-11927) | Fix "undefined reference to dlopen" error when compiling libhadooppipes | Major | build, native, tools | Xianyin Xin | Xianyin Xin | +| [YARN-3701](https://issues.apache.org/jira/browse/YARN-3701) | Isolating the error of generating a single app report when getting all apps from generic history service | Blocker | timelineserver | Zhijie Shen | Zhijie Shen | +| [YARN-3707](https://issues.apache.org/jira/browse/YARN-3707) | RM Web UI queue filter doesn't work | Blocker | . | Wangda Tan | Wangda Tan | +| [YARN-2238](https://issues.apache.org/jira/browse/YARN-2238) | filtering on UI sticks even if I move away from the page | Major | webapp | Sangjin Lee | Jian He | +| [HADOOP-8751](https://issues.apache.org/jira/browse/HADOOP-8751) | NPE in Token.toString() when Token is constructed using null identifier | Minor | security | Vlad Rozov | Kanaka Kumar Avvaru | +| [HADOOP-11969](https://issues.apache.org/jira/browse/HADOOP-11969) | ThreadLocal initialization in several classes is not thread safe | Critical | io | Sean Busbey | Sean Busbey | +| [YARN-3626](https://issues.apache.org/jira/browse/YARN-3626) | On Windows localized resources are not moved to the front of the classpath when they should be | Major | yarn | Craig Welch | Craig Welch | +| [HADOOP-9891](https://issues.apache.org/jira/browse/HADOOP-9891) | CLIMiniCluster instructions fail with MiniYarnCluster ClassNotFoundException | Minor | documentation | Steve Loughran | Darrell Taylor | +| [HDFS-8431](https://issues.apache.org/jira/browse/HDFS-8431) | hdfs crypto class not found in Windows | Critical | scripts | Sumana Sathish | Anu Engineer | +| [HADOOP-12004](https://issues.apache.org/jira/browse/HADOOP-12004) | test-patch breaks with reexec in certain situations | Critical | . | Allen Wittenauer | Sean Busbey | +| [YARN-3723](https://issues.apache.org/jira/browse/YARN-3723) | Need to clearly document primaryFilter and otherInfo value type | Critical | timelineserver | Zhijie Shen | Zhijie Shen | +| [HDFS-8407](https://issues.apache.org/jira/browse/HDFS-8407) | libhdfs hdfsListDirectory must set errno to 0 on success | Major | native | Juan Yu | Masatake Iwasaki | +| [HDFS-8429](https://issues.apache.org/jira/browse/HDFS-8429) | Avoid stuck threads if there is an error in DomainSocketWatcher that stops the thread | Major | . | zhouyingchao | zhouyingchao | +| [HADOOP-11959](https://issues.apache.org/jira/browse/HADOOP-11959) | WASB should configure client side socket timeout in storage client blob request options | Major | tools | Ivan Mitic | Ivan Mitic | +| [HADOOP-11934](https://issues.apache.org/jira/browse/HADOOP-11934) | Use of JavaKeyStoreProvider in LdapGroupsMapping causes infinite loop | Blocker | security | Mike Yoder | Larry McCay | +| [HDFS-7401](https://issues.apache.org/jira/browse/HDFS-7401) | Add block info to DFSInputStream' WARN message when it adds node to deadNodes | Minor | . | Ming Ma | Mohammad Arshad | +| [HADOOP-12042](https://issues.apache.org/jira/browse/HADOOP-12042) | Users may see TrashPolicy if hdfs dfs -rm is run | Major | . | Allen Wittenauer | J.Andreina | +| [HDFS-7609](https://issues.apache.org/jira/browse/HDFS-7609) | Avoid retry cache collision when Standby NameNode loading edits | Critical | namenode | Carrey Zhan | Ming Ma | +| [HADOOP-11885](https://issues.apache.org/jira/browse/HADOOP-11885) | hadoop-dist dist-layout-stitching.sh does not work with dash | Major | build | Andrew Wang | Andrew Wang | +| [YARN-3725](https://issues.apache.org/jira/browse/YARN-3725) | App submission via REST API is broken in secure mode due to Timeline DT service address is empty | Blocker | resourcemanager, timelineserver | Zhijie Shen | Zhijie Shen | +| [HADOOP-12037](https://issues.apache.org/jira/browse/HADOOP-12037) | Fix wrong classname in example configuration of hadoop-auth documentation | Trivial | documentation | Masatake Iwasaki | Masatake Iwasaki | +| [HDFS-8256](https://issues.apache.org/jira/browse/HDFS-8256) | fsck "-storagepolicies , -blockId ,-replicaDetails " options are missed out in usage and from documentation | Major | documentation | J.Andreina | J.Andreina | +| [HDFS-8486](https://issues.apache.org/jira/browse/HDFS-8486) | DN startup may cause severe data loss | Blocker | datanode | Daryn Sharp | Daryn Sharp | +| [HDFS-8386](https://issues.apache.org/jira/browse/HDFS-8386) | Improve synchronization of 'streamer' reference in DFSOutputStream | Major | hdfs-client | Rakesh R | Rakesh R | +| [HDFS-8513](https://issues.apache.org/jira/browse/HDFS-8513) | Rename BlockPlacementPolicyRackFaultTolarent to BlockPlacementPolicyRackFaultTolerant | Minor | namenode | Andrew Wang | Andrew Wang | +| [HADOOP-11991](https://issues.apache.org/jira/browse/HADOOP-11991) | test-patch.sh isn't re-executed even if smart-apply-patch.sh is modified | Major | test | Kengo Seki | Kengo Seki | +| [HDFS-8270](https://issues.apache.org/jira/browse/HDFS-8270) | create() always retried with hardcoded timeout when file already exists with open lease | Major | hdfs-client | Andrey Stepachev | J.Andreina | +| [HDFS-8470](https://issues.apache.org/jira/browse/HDFS-8470) | fsimage loading progress should update inode, delegation token and cache pool count. | Minor | namenode | tongshiquan | Surendra Singh Lilhore | +| [HDFS-8523](https://issues.apache.org/jira/browse/HDFS-8523) | Remove usage information on unsupported operation "fsck -showprogress" from branch-2 | Major | documentation | J.Andreina | J.Andreina | +| [HDFS-3716](https://issues.apache.org/jira/browse/HDFS-3716) | Purger should remove stale fsimage ckpt files | Minor | namenode | suja s | J.Andreina | +| [YARN-3751](https://issues.apache.org/jira/browse/YARN-3751) | TestAHSWebServices fails after YARN-3467 | Major | . | Zhijie Shen | Sunil G | +| [YARN-3585](https://issues.apache.org/jira/browse/YARN-3585) | NodeManager cannot exit on SHUTDOWN event triggered and NM recovery is enabled | Critical | . | Peng Zhang | Rohith Sharma K S | +| [MAPREDUCE-6374](https://issues.apache.org/jira/browse/MAPREDUCE-6374) | Distributed Cache File visibility should check permission of full path | Major | . | Chang Li | Chang Li | +| [YARN-3762](https://issues.apache.org/jira/browse/YARN-3762) | FairScheduler: CME on FSParentQueue#getQueueUserAclInfo | Critical | fairscheduler | Karthik Kambatla | Karthik Kambatla | +| [YARN-3749](https://issues.apache.org/jira/browse/YARN-3749) | We should make a copy of configuration when init MiniYARNCluster with multiple RMs | Major | . | Chun Chen | Chun Chen | +| [MAPREDUCE-5965](https://issues.apache.org/jira/browse/MAPREDUCE-5965) | Hadoop streaming throws error if list of input files is high. Error is: "error=7, Argument list too long at if number of input file is high" | Major | . | Arup Malakar | Wilfred Spiegelenburg | +| [HADOOP-12018](https://issues.apache.org/jira/browse/HADOOP-12018) | smart-apply-patch.sh fails if the patch edits CR+LF files and is created by 'git diff --no-prefix' | Minor | build | Akira Ajisaka | Kengo Seki | +| [HADOOP-12019](https://issues.apache.org/jira/browse/HADOOP-12019) | update BUILDING.txt to include python for 'mvn site' in windows | Major | . | Vinayakumar B | Vinayakumar B | +| [MAPREDUCE-6382](https://issues.apache.org/jira/browse/MAPREDUCE-6382) | Don't escape HTML links in Diagnostics in JHS job overview | Major | . | Siqi Li | Siqi Li | +| [HADOOP-12058](https://issues.apache.org/jira/browse/HADOOP-12058) | Fix dead links to DistCp and Hadoop Archives pages. | Minor | documentation, site | Kazuho Fujii | Kazuho Fujii | +| [HDFS-8463](https://issues.apache.org/jira/browse/HDFS-8463) | Calling DFSInputStream.seekToNewSource just after stream creation causes NullPointerException | Minor | . | Masatake Iwasaki | Masatake Iwasaki | +| [YARN-3764](https://issues.apache.org/jira/browse/YARN-3764) | CapacityScheduler should forbid moving LeafQueue from one parent to another | Blocker | . | Wangda Tan | Wangda Tan | +| [HADOOP-11994](https://issues.apache.org/jira/browse/HADOOP-11994) | smart-apply-patch wrongly assumes that git is infallible | Major | test | Allen Wittenauer | Kengo Seki | +| [HADOOP-11924](https://issues.apache.org/jira/browse/HADOOP-11924) | Tolerate JDK-8047340-related exceptions in Shell#isSetSidAvailable preventing class init | Major | . | Gera Shegalov | Tsuyoshi Ozawa | +| [YARN-3733](https://issues.apache.org/jira/browse/YARN-3733) | Fix DominantRC#compare() does not work as expected if cluster resource is empty | Blocker | resourcemanager | Bibin A Chundatt | Rohith Sharma K S | +| [MAPREDUCE-6377](https://issues.apache.org/jira/browse/MAPREDUCE-6377) | JHS sorting on state column not working in webUi | Minor | jobhistoryserver | Bibin A Chundatt | zhihai xu | +| [MAPREDUCE-6387](https://issues.apache.org/jira/browse/MAPREDUCE-6387) | Serialize the recently added Task#encryptedSpillKey field at the end | Minor | . | Arun Suresh | Arun Suresh | +| [HADOOP-12056](https://issues.apache.org/jira/browse/HADOOP-12056) | Use DirectoryStream in DiskChecker#checkDirs to detect errors when listing a directory | Major | util | zhihai xu | zhihai xu | +| [HDFS-8522](https://issues.apache.org/jira/browse/HDFS-8522) | Change heavily recorded NN logs from INFO to DEBUG level | Major | namenode | Xiaoyu Yao | Xiaoyu Yao | +| [YARN-3655](https://issues.apache.org/jira/browse/YARN-3655) | FairScheduler: potential livelock due to maxAMShare limitation and container reservation | Critical | fairscheduler | zhihai xu | zhihai xu | +| [HDFS-8539](https://issues.apache.org/jira/browse/HDFS-8539) | Hdfs doesnt have class 'debug' in windows | Major | scripts | Sumana Sathish | Anu Engineer | +| [YARN-3780](https://issues.apache.org/jira/browse/YARN-3780) | Should use equals when compare Resource in RMNodeImpl#ReconnectNodeTransition | Minor | resourcemanager | zhihai xu | zhihai xu | +| [YARN-3747](https://issues.apache.org/jira/browse/YARN-3747) | TestLocalDirsHandlerService should delete the created test directory logDir2 | Minor | test | David Moore | David Moore | +| [HDFS-8554](https://issues.apache.org/jira/browse/HDFS-8554) | TestDatanodeLayoutUpgrade fails on Windows. | Major | test | Chris Nauroth | Chris Nauroth | +| [YARN-3778](https://issues.apache.org/jira/browse/YARN-3778) | Fix Yarn resourcemanger CLI usage | Major | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [HADOOP-12054](https://issues.apache.org/jira/browse/HADOOP-12054) | RPC client should not retry for InvalidToken exceptions | Critical | ipc | Daryn Sharp | Varun Saxena | +| [HDFS-8552](https://issues.apache.org/jira/browse/HDFS-8552) | Fix hdfs CLI usage message for namenode and zkfc | Major | . | Xiaoyu Yao | Brahma Reddy Battula | +| [HADOOP-12073](https://issues.apache.org/jira/browse/HADOOP-12073) | Azure FileSystem PageBlobInputStream does not return -1 on EOF | Major | tools | Ivan Mitic | Ivan Mitic | +| [HDFS-8568](https://issues.apache.org/jira/browse/HDFS-8568) | TestClusterId#testFormatWithEmptyClusterIdOption is failing | Major | . | Rakesh R | Rakesh R | +| [HADOOP-7817](https://issues.apache.org/jira/browse/HADOOP-7817) | RawLocalFileSystem.append() should give FSDataOutputStream with accurate .getPos() | Major | fs | Kristofer Tomasette | Kanaka Kumar Avvaru | +| [MAPREDUCE-6350](https://issues.apache.org/jira/browse/MAPREDUCE-6350) | JobHistory doesn't support fully-functional search | Critical | jobhistoryserver | Siqi Li | Siqi Li | +| [MAPREDUCE-6389](https://issues.apache.org/jira/browse/MAPREDUCE-6389) | Fix BaileyBorweinPlouffe CLI usage message | Trivial | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [HADOOP-12052](https://issues.apache.org/jira/browse/HADOOP-12052) | IPC client downgrades all exception types to IOE, breaks callers trying to use them | Critical | . | Steve Loughran | Brahma Reddy Battula | +| [YARN-3785](https://issues.apache.org/jira/browse/YARN-3785) | Support for Resource as an argument during submitApp call in MockRM test class | Minor | resourcemanager | Sunil G | Sunil G | +| [HADOOP-12074](https://issues.apache.org/jira/browse/HADOOP-12074) | in Shell.java#runCommand() rethrow InterruptedException as InterruptedIOException | Minor | . | Lavkesh Lahngir | Lavkesh Lahngir | +| [HDFS-8566](https://issues.apache.org/jira/browse/HDFS-8566) | HDFS documentation about debug commands wrongly identifies them as "hdfs dfs" commands | Major | documentation | Surendra Singh Lilhore | Surendra Singh Lilhore | +| [HDFS-8583](https://issues.apache.org/jira/browse/HDFS-8583) | Document that NFS gateway does not work with rpcbind on SLES 11 | Major | documentation | Arpit Agarwal | Arpit Agarwal | +| [HDFS-8572](https://issues.apache.org/jira/browse/HDFS-8572) | DN always uses HTTP/localhost@REALM principals in SPNEGO | Blocker | . | Haohui Mai | Haohui Mai | +| [YARN-3794](https://issues.apache.org/jira/browse/YARN-3794) | TestRMEmbeddedElector fails because of ambiguous LOG reference | Major | test | Chengbing Liu | Chengbing Liu | +| [HDFS-8593](https://issues.apache.org/jira/browse/HDFS-8593) | Calculation of effective layout version mishandles comparison to current layout version in storage. | Major | namenode | Chris Nauroth | Chris Nauroth | +| [HDFS-8596](https://issues.apache.org/jira/browse/HDFS-8596) | TestDistributedFileSystem et al tests are broken in branch-2 due to incorrect setting of "datanode" attribute | Blocker | datanode | Yongjun Zhang | Yongjun Zhang | +| [HDFS-8595](https://issues.apache.org/jira/browse/HDFS-8595) | TestCommitBlockSynchronization fails in branch-2.7 | Major | test | Arpit Agarwal | Arpit Agarwal | +| [HDFS-8607](https://issues.apache.org/jira/browse/HDFS-8607) | TestFileCorruption doesn't work as expected | Major | test | Walter Su | Walter Su | +| [HADOOP-12001](https://issues.apache.org/jira/browse/HADOOP-12001) | Limiting LDAP search conflicts with posixGroup addition | Blocker | security | Patrick White | Patrick White | +| [HADOOP-12078](https://issues.apache.org/jira/browse/HADOOP-12078) | The default retry policy does not handle RetriableException correctly | Critical | ipc | Arpit Agarwal | Arpit Agarwal | +| [HDFS-8576](https://issues.apache.org/jira/browse/HDFS-8576) | Lease recovery should return true if the lease can be released and the file can be closed | Major | namenode | J.Andreina | J.Andreina | +| [HDFS-8592](https://issues.apache.org/jira/browse/HDFS-8592) | SafeModeException never get unwrapped | Major | . | Haohui Mai | Haohui Mai | +| [HADOOP-12095](https://issues.apache.org/jira/browse/HADOOP-12095) | org.apache.hadoop.fs.shell.TestCount fails | Major | test | Brahma Reddy Battula | Brahma Reddy Battula | +| [HDFS-4660](https://issues.apache.org/jira/browse/HDFS-4660) | Block corruption can happen during pipeline recovery | Blocker | datanode | Peng Zhang | Kihwal Lee | +| [HDFS-8548](https://issues.apache.org/jira/browse/HDFS-8548) | Minicluster throws NPE on shutdown | Major | . | Mike Drob | Surendra Singh Lilhore | +| [YARN-3714](https://issues.apache.org/jira/browse/YARN-3714) | AM proxy filter can not get RM webapp address from yarn.resourcemanager.hostname.rm-id | Minor | . | Masatake Iwasaki | Masatake Iwasaki | +| [HDFS-8551](https://issues.apache.org/jira/browse/HDFS-8551) | Fix hdfs datanode CLI usage message | Major | . | Xiaoyu Yao | Brahma Reddy Battula | +| [HADOOP-12076](https://issues.apache.org/jira/browse/HADOOP-12076) | Incomplete Cache Mechanism in CredentialProvider API | Major | security | Larry McCay | Larry McCay | +| [YARN-3617](https://issues.apache.org/jira/browse/YARN-3617) | Fix WindowsResourceCalculatorPlugin.getCpuFrequency() returning always -1 | Minor | . | Georg Berendt | J.Andreina | +| [YARN-3804](https://issues.apache.org/jira/browse/YARN-3804) | Both RM are on standBy state when kerberos user not in yarn.admin.acl | Critical | resourcemanager | Bibin A Chundatt | Varun Saxena | +| [HDFS-8446](https://issues.apache.org/jira/browse/HDFS-8446) | Separate safemode related operations in GetBlockLocations() | Minor | . | Haohui Mai | Haohui Mai | +| [HDFS-8615](https://issues.apache.org/jira/browse/HDFS-8615) | Correct HTTP method in WebHDFS document | Major | documentation | Akira Ajisaka | Brahma Reddy Battula | +| [MAPREDUCE-6373](https://issues.apache.org/jira/browse/MAPREDUCE-6373) | The logger reports total input paths but it is referring to input files | Trivial | . | Andi Chirita Amdocs | Bibin A Chundatt | +| [YARN-3824](https://issues.apache.org/jira/browse/YARN-3824) | Fix two minor nits in member variable properties of YarnConfiguration | Trivial | yarn | Ray Chiang | Ray Chiang | +| [HADOOP-12100](https://issues.apache.org/jira/browse/HADOOP-12100) | ImmutableFsPermission should not override applyUmask since that method doesn't modify the FsPermission | Major | . | Robert Kanter | Bibin A Chundatt | +| [YARN-3802](https://issues.apache.org/jira/browse/YARN-3802) | Two RMNodes for the same NodeId are used in RM sometimes after NM is reconnected. | Major | resourcemanager | zhihai xu | zhihai xu | +| [HDFS-8633](https://issues.apache.org/jira/browse/HDFS-8633) | Fix setting of dfs.datanode.readahead.bytes in hdfs-default.xml to match DFSConfigKeys | Minor | datanode | Ray Chiang | Ray Chiang | +| [HDFS-8626](https://issues.apache.org/jira/browse/HDFS-8626) | Reserved RBW space is not released if creation of RBW File fails | Blocker | . | Kanaka Kumar Avvaru | Kanaka Kumar Avvaru | +| [MAPREDUCE-6405](https://issues.apache.org/jira/browse/MAPREDUCE-6405) | NullPointerException in App Attempts page | Major | . | Siqi Li | Siqi Li | +| [HADOOP-12103](https://issues.apache.org/jira/browse/HADOOP-12103) | Small refactoring of DelegationTokenAuthenticationFilter to allow code sharing | Minor | security | Yongjun Zhang | Yongjun Zhang | +| [HDFS-8337](https://issues.apache.org/jira/browse/HDFS-8337) | Accessing httpfs via webhdfs doesn't work from a jar with kerberos | Major | security, webhdfs | Yongjun Zhang | Yongjun Zhang | +| [MAPREDUCE-6403](https://issues.apache.org/jira/browse/MAPREDUCE-6403) | Fix typo in the usage of NNBench | Trivial | documentation | Akira Ajisaka | Jagadesh Kiran N | +| [HDFS-8480](https://issues.apache.org/jira/browse/HDFS-8480) | Fix performance and timeout issues in HDFS-7929 by using hard-links to preserve old edit logs instead of copying them | Critical | . | Zhe Zhang | Zhe Zhang | +| [MAPREDUCE-5948](https://issues.apache.org/jira/browse/MAPREDUCE-5948) | org.apache.hadoop.mapred.LineRecordReader does not handle multibyte record delimiters well | Critical | . | Kris Geusebroek | Akira Ajisaka | +| [HDFS-8542](https://issues.apache.org/jira/browse/HDFS-8542) | WebHDFS getHomeDirectory behavior does not match specification | Major | webhdfs | Jakob Homan | Kanaka Kumar Avvaru | +| [YARN-3842](https://issues.apache.org/jira/browse/YARN-3842) | NMProxy should retry on NMNotYetReadyException | Critical | . | Karthik Kambatla | Robert Kanter | +| [YARN-3835](https://issues.apache.org/jira/browse/YARN-3835) | hadoop-yarn-server-resourcemanager test package bundles core-site.xml, yarn-site.xml | Minor | resourcemanager | Vamsee Yarlagadda | Vamsee Yarlagadda | +| [MAPREDUCE-6410](https://issues.apache.org/jira/browse/MAPREDUCE-6410) | Aggregated Logs Deletion doesnt work after refreshing Log Retention Settings in secure cluster | Critical | . | Zhang Wei | Varun Saxena | +| [MAPREDUCE-6400](https://issues.apache.org/jira/browse/MAPREDUCE-6400) | Multiple shuffle transfer fails because input is closed too early | Blocker | task | Akira Ajisaka | Brahma Reddy Battula | +| [YARN-3809](https://issues.apache.org/jira/browse/YARN-3809) | Failed to launch new attempts because ApplicationMasterLauncher's threads all hang | Major | resourcemanager | Jun Gong | Jun Gong | +| [YARN-3832](https://issues.apache.org/jira/browse/YARN-3832) | Resource Localization fails on a cluster due to existing cache directories | Critical | nodemanager | Ranga Swamy | Brahma Reddy Battula | +| [YARN-3790](https://issues.apache.org/jira/browse/YARN-3790) | usedResource from rootQueue metrics may get stale data for FS scheduler after recovering the container | Major | fairscheduler, test | Rohith Sharma K S | zhihai xu | +| [HADOOP-11958](https://issues.apache.org/jira/browse/HADOOP-11958) | MetricsSystemImpl fails to show backtrace when an error occurs | Major | . | Jason Lowe | Jason Lowe | +| [HDFS-8646](https://issues.apache.org/jira/browse/HDFS-8646) | Prune cached replicas from DatanodeDescriptor state on replica invalidation | Major | caching | Andrew Wang | Andrew Wang | +| [YARN-3826](https://issues.apache.org/jira/browse/YARN-3826) | Race condition in ResourceTrackerService leads to wrong diagnostics messages | Major | resourcemanager | Chengbing Liu | Chengbing Liu | +| [YARN-3745](https://issues.apache.org/jira/browse/YARN-3745) | SerializedException should also try to instantiate internal exception with the default constructor | Major | . | Lavkesh Lahngir | Lavkesh Lahngir | +| [MAPREDUCE-6413](https://issues.apache.org/jira/browse/MAPREDUCE-6413) | TestLocalJobSubmission is failing with unknown host | Major | test | Jason Lowe | zhihai xu | +| [HDFS-8665](https://issues.apache.org/jira/browse/HDFS-8665) | Fix replication check in DFSTestUtils#waitForReplication | Trivial | test | Andrew Wang | Andrew Wang | +| [HADOOP-8151](https://issues.apache.org/jira/browse/HADOOP-8151) | Error handling in snappy decompressor throws invalid exceptions | Major | io, native | Todd Lipcon | Matt Foley | +| [YARN-3850](https://issues.apache.org/jira/browse/YARN-3850) | NM fails to read files from full disks which can lead to container logs being lost and other issues | Blocker | log-aggregation, nodemanager | Varun Saxena | Varun Saxena | +| [HDFS-8656](https://issues.apache.org/jira/browse/HDFS-8656) | Preserve compatibility of ClientProtocol#rollingUpgrade after finalization | Critical | rolling upgrades | Andrew Wang | Andrew Wang | +| [YARN-3859](https://issues.apache.org/jira/browse/YARN-3859) | LeafQueue doesn't print user properly for application add | Minor | capacityscheduler | Devaraj K | Varun Saxena | +| [HDFS-8681](https://issues.apache.org/jira/browse/HDFS-8681) | BlockScanner is incorrectly disabled by default | Blocker | datanode | Andrew Wang | Arpit Agarwal | +| [YARN-3860](https://issues.apache.org/jira/browse/YARN-3860) | rmadmin -transitionToActive should check the state of non-target node | Major | resourcemanager | Masatake Iwasaki | Masatake Iwasaki | +| [HDFS-8586](https://issues.apache.org/jira/browse/HDFS-8586) | Dead Datanode is allocated for write when client is from deadnode | Critical | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [HADOOP-12119](https://issues.apache.org/jira/browse/HADOOP-12119) | hadoop fs -expunge does not work for federated namespace | Major | . | Vrushali C | J.Andreina | +| [HDFS-8628](https://issues.apache.org/jira/browse/HDFS-8628) | Update missing command option for fetchdt | Major | documentation | J.Andreina | J.Andreina | +| [YARN-3695](https://issues.apache.org/jira/browse/YARN-3695) | ServerProxy (NMProxy, etc.) shouldn't retry forever for non network exception. | Major | . | Junping Du | Raju Bairishetti | +| [HADOOP-12089](https://issues.apache.org/jira/browse/HADOOP-12089) | StorageException complaining " no lease ID" when updating FolderLastModifiedTime in WASB | Major | tools | Duo Xu | Duo Xu | +| [YARN-3770](https://issues.apache.org/jira/browse/YARN-3770) | SerializedException should also handle java.lang.Error | Major | . | Lavkesh Lahngir | Lavkesh Lahngir | +| [HDFS-8687](https://issues.apache.org/jira/browse/HDFS-8687) | Remove the duplicate usage message from Dfsck.java | Major | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [HADOOP-12107](https://issues.apache.org/jira/browse/HADOOP-12107) | long running apps may have a huge number of StatisticsData instances under FileSystem | Critical | fs | Sangjin Lee | Sangjin Lee | +| [HDFS-8579](https://issues.apache.org/jira/browse/HDFS-8579) | Update HDFS usage with missing options | Minor | . | J.Andreina | J.Andreina | +| [HADOOP-12154](https://issues.apache.org/jira/browse/HADOOP-12154) | FileSystem#getUsed() returns the file length only from root '/' | Major | . | tongshiquan | J.Andreina | +| [YARN-3768](https://issues.apache.org/jira/browse/YARN-3768) | ArrayIndexOutOfBoundsException with empty environment variables | Major | yarn | Joe Ferner | zhihai xu | +| [HADOOP-10798](https://issues.apache.org/jira/browse/HADOOP-10798) | globStatus() should always return a sorted list of files | Minor | . | Felix Borchers | Colin P. McCabe | +| [HADOOP-12124](https://issues.apache.org/jira/browse/HADOOP-12124) | Add HTrace support for FsShell | Major | . | Colin P. McCabe | Colin P. McCabe | +| [HADOOP-12159](https://issues.apache.org/jira/browse/HADOOP-12159) | Move DistCpUtils#compareFs() to org.apache.hadoop.fs.FileUtil and fix for HA namespaces | Major | . | Ray Chiang | Ray Chiang | +| [MAPREDUCE-6121](https://issues.apache.org/jira/browse/MAPREDUCE-6121) | JobResourceUpdater#compareFs() doesn't handle HA namespaces | Major | mrv2 | Thomas Graves | Ray Chiang | +| [HADOOP-12116](https://issues.apache.org/jira/browse/HADOOP-12116) | Fix unrecommended syntax usages in hadoop/hdfs/yarn script for cygwin in branch-2 | Major | scripts | Li Lu | Li Lu | +| [HADOOP-12164](https://issues.apache.org/jira/browse/HADOOP-12164) | Fix TestMove and TestFsShellReturnCode failed to get command name using reflection. | Minor | . | Lei (Eddy) Xu | Lei (Eddy) Xu | +| [YARN-3823](https://issues.apache.org/jira/browse/YARN-3823) | Fix mismatch in default values for yarn.scheduler.maximum-allocation-vcores property | Minor | . | Ray Chiang | Ray Chiang | +| [YARN-3830](https://issues.apache.org/jira/browse/YARN-3830) | AbstractYarnScheduler.createReleaseCache may try to clean a null attempt | Major | scheduler | nijel | nijel | +| [HDFS-8706](https://issues.apache.org/jira/browse/HDFS-8706) | Fix typo in datanode startup options in HDFSCommands.html | Major | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [MAPREDUCE-6420](https://issues.apache.org/jira/browse/MAPREDUCE-6420) | Interrupted Exception in LocalContainerLauncher should be logged in warn/info level | Major | . | Chang Li | Chang Li | +| [MAPREDUCE-6418](https://issues.apache.org/jira/browse/MAPREDUCE-6418) | MRApp should not shutdown LogManager during shutdown | Major | test | Chang Li | Chang Li | +| [YARN-3793](https://issues.apache.org/jira/browse/YARN-3793) | Several NPEs when deleting local files on NM recovery | Major | nodemanager | Karthik Kambatla | Varun Saxena | +| [HDFS-8666](https://issues.apache.org/jira/browse/HDFS-8666) | speedup TestMover | Major | test | Walter Su | Walter Su | +| [HADOOP-12171](https://issues.apache.org/jira/browse/HADOOP-12171) | Shorten overly-long htrace span names for server | Major | tracing | Colin P. McCabe | Colin P. McCabe | +| [YARN-3875](https://issues.apache.org/jira/browse/YARN-3875) | FSSchedulerNode#reserveResource() doesn't print Application Id properly in log | Minor | . | Bibin A Chundatt | Bibin A Chundatt | +| [YARN-3508](https://issues.apache.org/jira/browse/YARN-3508) | Prevent processing preemption events on the main RM dispatcher | Major | resourcemanager, scheduler | Jason Lowe | Varun Saxena | +| [HADOOP-12173](https://issues.apache.org/jira/browse/HADOOP-12173) | NetworkTopology#add calls NetworkTopology#toString always | Major | . | Inigo Goiri | Inigo Goiri | +| [HDFS-8577](https://issues.apache.org/jira/browse/HDFS-8577) | Avoid retrying to recover lease on a file which does not exist | Major | . | J.Andreina | J.Andreina | +| [YARN-3882](https://issues.apache.org/jira/browse/YARN-3882) | AggregatedLogFormat should close aclScanner and ownerScanner after create them. | Minor | nodemanager | zhihai xu | zhihai xu | +| [MAPREDUCE-6425](https://issues.apache.org/jira/browse/MAPREDUCE-6425) | ShuffleHandler passes wrong "base" parameter to getMapOutputInfo if mapId is not in the cache. | Major | mrv2, nodemanager | zhihai xu | zhihai xu | +| [HADOOP-12186](https://issues.apache.org/jira/browse/HADOOP-12186) | ActiveStandbyElector shouldn't call monitorLockNodeAsync multiple times | Major | ha | zhihai xu | zhihai xu | +| [HDFS-8686](https://issues.apache.org/jira/browse/HDFS-8686) | WebHdfsFileSystem#getXAttr(Path p, final String name) doesn't work if namespace is in capitals | Major | webhdfs | Jagadesh Kiran N | Kanaka Kumar Avvaru | +| [YARN-3837](https://issues.apache.org/jira/browse/YARN-3837) | javadocs of TimelineAuthenticationFilterInitializer give wrong prefix for auth options | Minor | timelineserver | Steve Loughran | Bibin A Chundatt | +| [HADOOP-12117](https://issues.apache.org/jira/browse/HADOOP-12117) | Potential NPE from Configuration#loadProperty with allowNullValueProperties set. | Minor | conf | zhihai xu | zhihai xu | +| [MAPREDUCE-6038](https://issues.apache.org/jira/browse/MAPREDUCE-6038) | A boolean may be set error in the Word Count v2.0 in MapReduce Tutorial | Minor | . | Pei Ma | Tsuyoshi Ozawa | +| [YARN-3892](https://issues.apache.org/jira/browse/YARN-3892) | NPE on RMStateStore#serviceStop when CapacityScheduler#serviceInit fails | Minor | . | Bibin A Chundatt | Bibin A Chundatt | +| [YARN-3690](https://issues.apache.org/jira/browse/YARN-3690) | [JDK8] 'mvn site' fails | Major | api, site | Akira Ajisaka | Brahma Reddy Battula | +| [HADOOP-12202](https://issues.apache.org/jira/browse/HADOOP-12202) | releasedocmaker drops missing component and assignee entries | Blocker | yetus | Allen Wittenauer | Allen Wittenauer | +| [HDFS-8642](https://issues.apache.org/jira/browse/HDFS-8642) | Make TestFileTruncate more reliable | Minor | . | Rakesh R | Rakesh R | +| [HADOOP-11878](https://issues.apache.org/jira/browse/HADOOP-11878) | FileContext.java # fixRelativePart should check for not null for a more informative exception | Major | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [HADOOP-12201](https://issues.apache.org/jira/browse/HADOOP-12201) | Add tracing to FileSystem#createFileSystem and Globber#glob | Major | . | Colin P. McCabe | Colin P. McCabe | +| [HADOOP-12200](https://issues.apache.org/jira/browse/HADOOP-12200) | TestCryptoStreamsWithOpensslAesCtrCryptoCodec should be skipped in non-native profile | Minor | test | Masatake Iwasaki | Masatake Iwasaki | +| [MAPREDUCE-6426](https://issues.apache.org/jira/browse/MAPREDUCE-6426) | TestShuffleHandler#testGetMapOutputInfo is failing | Major | test | Devaraj K | zhihai xu | +| [HDFS-8729](https://issues.apache.org/jira/browse/HDFS-8729) | Fix testTruncateWithDataNodesRestartImmediately occasionally failed | Minor | . | Walter Su | Walter Su | +| [YARN-3888](https://issues.apache.org/jira/browse/YARN-3888) | ApplicationMaster link is broken in RM WebUI when appstate is NEW | Minor | resourcemanager | Bibin A Chundatt | Bibin A Chundatt | +| [HDFS-8749](https://issues.apache.org/jira/browse/HDFS-8749) | Fix findbugs warning in BlockManager.java | Minor | . | Akira Ajisaka | Brahma Reddy Battula | +| [HDFS-2956](https://issues.apache.org/jira/browse/HDFS-2956) | calling fetchdt without a --renewer argument throws NPE | Major | security | Todd Lipcon | Vinayakumar B | +| [HDFS-8751](https://issues.apache.org/jira/browse/HDFS-8751) | Remove setBlocks API from INodeFile and misc code cleanup | Major | namenode | Zhe Zhang | Zhe Zhang | +| [YARN-3849](https://issues.apache.org/jira/browse/YARN-3849) | Too much of preemption activity causing continuos killing of containers across queues | Critical | capacityscheduler | Sunil G | Sunil G | +| [YARN-3917](https://issues.apache.org/jira/browse/YARN-3917) | getResourceCalculatorPlugin for the default should intercept all exceptions | Major | . | Gera Shegalov | Gera Shegalov | +| [YARN-3894](https://issues.apache.org/jira/browse/YARN-3894) | RM startup should fail for wrong CS xml NodeLabel capacity configuration | Critical | capacityscheduler | Bibin A Chundatt | Bibin A Chundatt | +| [MAPREDUCE-6421](https://issues.apache.org/jira/browse/MAPREDUCE-6421) | Fix findbugs warning in RMContainerAllocator.reduceNodeLabelExpression | Major | . | Ray Chiang | Brahma Reddy Battula | +| [HADOOP-12191](https://issues.apache.org/jira/browse/HADOOP-12191) | Bzip2Factory is not thread safe | Major | io | Jason Lowe | Brahma Reddy Battula | +| [HDFS-7608](https://issues.apache.org/jira/browse/HDFS-7608) | hdfs dfsclient newConnectedPeer has no write timeout | Major | fuse-dfs, hdfs-client | zhangshilong | Xiaoyu Yao | +| [HADOOP-12153](https://issues.apache.org/jira/browse/HADOOP-12153) | ByteBufferReadable doesn't declare @InterfaceAudience and @InterfaceStability | Minor | fs | Steve Loughran | Brahma Reddy Battula | +| [HDFS-8778](https://issues.apache.org/jira/browse/HDFS-8778) | TestBlockReportRateLimiting#testLeaseExpiration can deadlock | Major | test | Arpit Agarwal | Arpit Agarwal | +| [HADOOP-10615](https://issues.apache.org/jira/browse/HADOOP-10615) | FileInputStream in JenkinsHash#main() is never closed | Minor | . | Ted Yu | Chen He | +| [HADOOP-12240](https://issues.apache.org/jira/browse/HADOOP-12240) | Fix tests requiring native library to be skipped in non-native profile | Minor | test | Masatake Iwasaki | Masatake Iwasaki | +| [YARN-3805](https://issues.apache.org/jira/browse/YARN-3805) | Update the documentation of Disk Checker based on YARN-90 | Minor | documentation | Masatake Iwasaki | Masatake Iwasaki | +| [HDFS-8767](https://issues.apache.org/jira/browse/HDFS-8767) | RawLocalFileSystem.listStatus() returns null for UNIX pipefile | Critical | . | Haohui Mai | Kanaka Kumar Avvaru | +| [YARN-3885](https://issues.apache.org/jira/browse/YARN-3885) | ProportionalCapacityPreemptionPolicy doesn't preempt if queue is more than 2 level | Blocker | yarn | Ajith S | Ajith S | +| [YARN-3453](https://issues.apache.org/jira/browse/YARN-3453) | Fair Scheduler: Parts of preemption logic uses DefaultResourceCalculator even in DRF mode causing thrashing | Major | fairscheduler | Ashwin Shankar | Arun Suresh | +| [YARN-3535](https://issues.apache.org/jira/browse/YARN-3535) | Scheduler must re-request container resources when RMContainer transitions from ALLOCATED to KILLED | Critical | capacityscheduler, fairscheduler, resourcemanager | Peng Zhang | Peng Zhang | +| [YARN-3905](https://issues.apache.org/jira/browse/YARN-3905) | Application History Server UI NPEs when accessing apps run after RM restart | Major | timelineserver | Eric Payne | Eric Payne | +| [HADOOP-12235](https://issues.apache.org/jira/browse/HADOOP-12235) | hadoop-openstack junit & mockito dependencies should be "provided" | Minor | build, fs/swift | Steve Loughran | Ted Yu | +| [HADOOP-12209](https://issues.apache.org/jira/browse/HADOOP-12209) | Comparable type should be in FileStatus | Minor | fs | Yong Zhang | Yong Zhang | +| [HADOOP-12088](https://issues.apache.org/jira/browse/HADOOP-12088) | KMSClientProvider uses equalsIgnoreCase("application/json") | Major | kms | Steve Loughran | Brahma Reddy Battula | +| [HADOOP-12051](https://issues.apache.org/jira/browse/HADOOP-12051) | ProtobufRpcEngine.invoke() should use Exception.toString() over getMessage in logging/span events | Minor | ipc | Steve Loughran | Varun Saxena | +| [HADOOP-12237](https://issues.apache.org/jira/browse/HADOOP-12237) | releasedocmaker.py doesn't work behind a proxy | Major | yetus | Tsuyoshi Ozawa | Tsuyoshi Ozawa | +| [HDFS-7582](https://issues.apache.org/jira/browse/HDFS-7582) | Enforce maximum number of ACL entries separately per access and default. | Major | namenode | Vinayakumar B | Vinayakumar B | +| [HDFS-8773](https://issues.apache.org/jira/browse/HDFS-8773) | Few FSNamesystem metrics are not documented in the Metrics page | Major | documentation | Rakesh R | Rakesh R | +| [YARN-3878](https://issues.apache.org/jira/browse/YARN-3878) | AsyncDispatcher can hang while stopping if it is configured for draining events on stop | Critical | . | Varun Saxena | Varun Saxena | +| [HDFS-7728](https://issues.apache.org/jira/browse/HDFS-7728) | Avoid updating quota usage while loading edits | Major | . | Jing Zhao | Jing Zhao | +| [HADOOP-11962](https://issues.apache.org/jira/browse/HADOOP-11962) | Sasl message with MD5 challenge text shouldn't be LOG out even in debug level. | Critical | ipc, security | Junping Du | Junping Du | +| [HADOOP-12017](https://issues.apache.org/jira/browse/HADOOP-12017) | Hadoop archives command should use configurable replication factor when closing | Major | . | Zhe Zhang | Bibin A Chundatt | +| [HADOOP-12239](https://issues.apache.org/jira/browse/HADOOP-12239) | StorageException complaining " no lease ID" when updating FolderLastModifiedTime in WASB | Major | fs/azure, tools | Duo Xu | Duo Xu | +| [YARN-3932](https://issues.apache.org/jira/browse/YARN-3932) | SchedulerApplicationAttempt#getResourceUsageReport and UserInfo should based on total-used-resources | Major | resourcemanager | Bibin A Chundatt | Bibin A Chundatt | +| [YARN-3954](https://issues.apache.org/jira/browse/YARN-3954) | TestYarnConfigurationFields#testCompareConfigurationClassAgainstXml fails in trunk | Major | . | Varun Saxena | Varun Saxena | +| [YARN-2019](https://issues.apache.org/jira/browse/YARN-2019) | Retrospect on decision of making RM crashed if any exception throw in ZKRMStateStore | Critical | . | Junping Du | Jian He | +| [HDFS-8797](https://issues.apache.org/jira/browse/HDFS-8797) | WebHdfsFileSystem creates too many connections for pread | Major | webhdfs | Jing Zhao | Jing Zhao | +| [YARN-3941](https://issues.apache.org/jira/browse/YARN-3941) | Proportional Preemption policy should try to avoid sending duplicate PREEMPT\_CONTAINER event to scheduler | Major | capacityscheduler | Sunil G | Sunil G | +| [YARN-3900](https://issues.apache.org/jira/browse/YARN-3900) | Protobuf layout of yarn\_security\_token causes errors in other protos that include it | Major | . | Anubhav Dhoot | Anubhav Dhoot | +| [YARN-3845](https://issues.apache.org/jira/browse/YARN-3845) | Scheduler page does not render RGBA color combinations in IE11 | Minor | . | Jagadesh Kiran N | Mohammad Shahid Khan | +| [HDFS-8806](https://issues.apache.org/jira/browse/HDFS-8806) | Inconsistent metrics: number of missing blocks with replication factor 1 not properly cleared | Major | . | Zhe Zhang | Zhe Zhang | +| [YARN-3967](https://issues.apache.org/jira/browse/YARN-3967) | Fetch the application report from the AHS if the RM does not know about it | Major | . | Mit Desai | Mit Desai | +| [YARN-3957](https://issues.apache.org/jira/browse/YARN-3957) | FairScheduler NPE In FairSchedulerQueueInfo causing scheduler page to return 500 | Major | fairscheduler | Anubhav Dhoot | Anubhav Dhoot | +| [YARN-3925](https://issues.apache.org/jira/browse/YARN-3925) | ContainerLogsUtils#getContainerLogFile fails to read container log files from full disks. | Critical | nodemanager | zhihai xu | zhihai xu | +| [YARN-3973](https://issues.apache.org/jira/browse/YARN-3973) | Recent changes to application priority management break reservation system from YARN-1051 | Major | resourcemanager | Carlo Curino | Carlo Curino | +| [YARN-3958](https://issues.apache.org/jira/browse/YARN-3958) | TestYarnConfigurationFields should be moved to hadoop-yarn-api module | Major | . | Varun Saxena | Varun Saxena | +| [HDFS-8810](https://issues.apache.org/jira/browse/HDFS-8810) | Correct assertions in TestDFSInotifyEventInputStream class. | Minor | test | Surendra Singh Lilhore | Surendra Singh Lilhore | +| [HDFS-8785](https://issues.apache.org/jira/browse/HDFS-8785) | TestDistributedFileSystem is failing in trunk | Major | test | Arpit Agarwal | Xiaoyu Yao | +| [YARN-2194](https://issues.apache.org/jira/browse/YARN-2194) | Cgroups cease to work in RHEL7 | Critical | nodemanager | Wei Yan | Wei Yan | +| [YARN-3846](https://issues.apache.org/jira/browse/YARN-3846) | RM Web UI queue filter is not working | Major | yarn | Mohammad Shahid Khan | Mohammad Shahid Khan | +| [HADOOP-12245](https://issues.apache.org/jira/browse/HADOOP-12245) | References to misspelled REMAINING\_QUATA in FileSystemShell.md | Minor | documentation | Gera Shegalov | Gabor Liptak | +| [YARN-3982](https://issues.apache.org/jira/browse/YARN-3982) | container-executor parsing of container-executor.cfg broken in trunk and branch-2 | Blocker | nodemanager | Varun Vasudev | Varun Vasudev | +| [HADOOP-12175](https://issues.apache.org/jira/browse/HADOOP-12175) | FsShell must load SpanReceiverHost to support tracing | Major | tracing | Masatake Iwasaki | Masatake Iwasaki | +| [HDFS-8670](https://issues.apache.org/jira/browse/HDFS-8670) | Better to exclude decommissioned nodes for namenode NodeUsage JMX | Major | . | Ming Ma | J.Andreina | +| [HADOOP-10945](https://issues.apache.org/jira/browse/HADOOP-10945) | 4-digit octal umask permissions throws a parse error | Major | fs | Jason Lowe | Chang Li | +| [YARN-3919](https://issues.apache.org/jira/browse/YARN-3919) | NPEs' while stopping service after exception during CommonNodeLabelsManager#start | Trivial | resourcemanager | Varun Saxena | Varun Saxena | +| [YARN-3963](https://issues.apache.org/jira/browse/YARN-3963) | AddNodeLabel on duplicate label addition shows success | Minor | . | Bibin A Chundatt | Bibin A Chundatt | +| [MAPREDUCE-6433](https://issues.apache.org/jira/browse/MAPREDUCE-6433) | launchTime may be negative | Major | jobhistoryserver, mrv2 | Allen Wittenauer | zhihai xu | +| [YARN-3990](https://issues.apache.org/jira/browse/YARN-3990) | AsyncDispatcher may overloaded with RMAppNodeUpdateEvent when Node is connected/disconnected | Critical | resourcemanager | Rohith Sharma K S | Bibin A Chundatt | +| [HDFS-6860](https://issues.apache.org/jira/browse/HDFS-6860) | BlockStateChange logs are too noisy | Major | namenode | Arpit Agarwal | Chang Li | +| [HADOOP-12268](https://issues.apache.org/jira/browse/HADOOP-12268) | AbstractContractAppendTest#testRenameFileBeingAppended misses rename operation. | Major | test | zhihai xu | zhihai xu | +| [HDFS-8847](https://issues.apache.org/jira/browse/HDFS-8847) | change TestHDFSContractAppend to not override testRenameFileBeingAppended method. | Major | test | zhihai xu | zhihai xu | +| [HDFS-8850](https://issues.apache.org/jira/browse/HDFS-8850) | VolumeScanner thread exits with exception if there is no block pool to be scanned but there are suspicious blocks | Major | datanode | Colin P. McCabe | Colin P. McCabe | +| [HDFS-8844](https://issues.apache.org/jira/browse/HDFS-8844) | TestHDFSCLI does not cleanup the test directory | Minor | test | Akira Ajisaka | Masatake Iwasaki | +| [HADOOP-12274](https://issues.apache.org/jira/browse/HADOOP-12274) | Remove direct download link from BUILDING.txt | Minor | documentation | Caleb Severn | Caleb Severn | +| [HADOOP-12302](https://issues.apache.org/jira/browse/HADOOP-12302) | Fix native compilation on Windows after HADOOP-7824 | Blocker | . | Vinayakumar B | Vinayakumar B | +| [YARN-3983](https://issues.apache.org/jira/browse/YARN-3983) | Make CapacityScheduler to easier extend application allocation logic | Major | . | Wangda Tan | Wangda Tan | +| [HADOOP-12304](https://issues.apache.org/jira/browse/HADOOP-12304) | Applications using FileContext fail with the default file system configured to be wasb/s3/etc. | Blocker | fs | Chris Nauroth | Chris Nauroth | +| [HADOOP-11932](https://issues.apache.org/jira/browse/HADOOP-11932) | MetricsSinkAdapter hangs when being stopped | Critical | . | Jian He | Brahma Reddy Battula | +| [HDFS-8856](https://issues.apache.org/jira/browse/HDFS-8856) | Make LeaseManager#countPath O(1) | Major | namenode | Arpit Agarwal | Arpit Agarwal | +| [HDFS-8772](https://issues.apache.org/jira/browse/HDFS-8772) | Fix TestStandbyIsHot#testDatanodeRestarts which occasionally fails | Major | . | Walter Su | Walter Su | +| [YARN-3966](https://issues.apache.org/jira/browse/YARN-3966) | Fix excessive loggings in CapacityScheduler | Major | . | Jian He | Jian He | +| [HDFS-8866](https://issues.apache.org/jira/browse/HDFS-8866) | Typo in docs: Rumtime -\> Runtime | Trivial | documentation, webhdfs | Jakob Homan | Gabor Liptak | +| [YARN-3999](https://issues.apache.org/jira/browse/YARN-3999) | RM hangs on draining events | Major | . | Jian He | Jian He | +| [YARN-4026](https://issues.apache.org/jira/browse/YARN-4026) | FiCaSchedulerApp: ContainerAllocator should be able to choose how to order pending resource requests | Major | . | Wangda Tan | Wangda Tan | +| [HDFS-8879](https://issues.apache.org/jira/browse/HDFS-8879) | Quota by storage type usage incorrectly initialized upon namenode restart | Major | namenode | Kihwal Lee | Xiaoyu Yao | +| [HADOOP-12258](https://issues.apache.org/jira/browse/HADOOP-12258) | Need translate java.nio.file.NoSuchFileException to FileNotFoundException to avoid regression | Critical | fs | zhihai xu | zhihai xu | +| [YARN-4005](https://issues.apache.org/jira/browse/YARN-4005) | Completed container whose app is finished is not removed from NMStateStore | Major | . | Jun Gong | Jun Gong | +| [YARN-4047](https://issues.apache.org/jira/browse/YARN-4047) | ClientRMService getApplications has high scheduler lock contention | Major | resourcemanager | Jason Lowe | Jason Lowe | +| [YARN-3987](https://issues.apache.org/jira/browse/YARN-3987) | am container complete msg ack to NM once RM receive it | Major | resourcemanager | sandflee | sandflee | +| [HADOOP-12322](https://issues.apache.org/jira/browse/HADOOP-12322) | typos in rpcmetrics.java | Trivial | ipc | Anu Engineer | Anu Engineer | +| [HDFS-8565](https://issues.apache.org/jira/browse/HDFS-8565) | Typo in dfshealth.html - "Decomissioning" | Trivial | . | nijel | nijel | +| [MAPREDUCE-5817](https://issues.apache.org/jira/browse/MAPREDUCE-5817) | Mappers get rescheduled on node transition even after all reducers are completed | Major | applicationmaster | Sangjin Lee | Sangjin Lee | +| [HDFS-8891](https://issues.apache.org/jira/browse/HDFS-8891) | HDFS concat should keep srcs order | Blocker | . | Yong Zhang | Yong Zhang | +| [MAPREDUCE-6439](https://issues.apache.org/jira/browse/MAPREDUCE-6439) | AM may fail instead of retrying if RM shuts down during the allocate call | Critical | . | Anubhav Dhoot | Anubhav Dhoot | +| [HDFS-8845](https://issues.apache.org/jira/browse/HDFS-8845) | DiskChecker should not traverse the entire tree | Major | . | Chang Li | Chang Li | +| [HDFS-8852](https://issues.apache.org/jira/browse/HDFS-8852) | HDFS architecture documentation of version 2.x is outdated about append write support | Major | documentation | Hong Dai Thanh | Ajith S | +| [YARN-3857](https://issues.apache.org/jira/browse/YARN-3857) | Memory leak in ResourceManager with SIMPLE mode | Critical | resourcemanager | mujunchao | mujunchao | +| [YARN-4028](https://issues.apache.org/jira/browse/YARN-4028) | AppBlock page key update and diagnostics value null on recovery | Minor | . | Bibin A Chundatt | Bibin A Chundatt | +| [HDFS-8908](https://issues.apache.org/jira/browse/HDFS-8908) | TestAppendSnapshotTruncate may fail with IOException: Failed to replace a bad datanode | Minor | test | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze | +| [HDFS-8867](https://issues.apache.org/jira/browse/HDFS-8867) | Enable optimized block reports | Major | . | Rushabh S Shah | Daryn Sharp | +| [HADOOP-12317](https://issues.apache.org/jira/browse/HADOOP-12317) | Applications fail on NM restart on some linux distro because NM container recovery declares AM container as LOST | Critical | . | Anubhav Dhoot | Anubhav Dhoot | +| [HDFS-8863](https://issues.apache.org/jira/browse/HDFS-8863) | The remaining space check in BlockPlacementPolicyDefault is flawed | Critical | . | Kihwal Lee | Kihwal Lee | +| [HDFS-8922](https://issues.apache.org/jira/browse/HDFS-8922) | Link the native\_mini\_dfs test library with libdl, since IBM Java requires it | Major | build | Ayappan | Ayappan | +| [HDFS-8809](https://issues.apache.org/jira/browse/HDFS-8809) | HDFS fsck reports under construction blocks as "CORRUPT" | Major | tools | Sudhir Prakash | Jing Zhao | +| [MAPREDUCE-6454](https://issues.apache.org/jira/browse/MAPREDUCE-6454) | MapReduce doesn't set the HADOOP\_CLASSPATH for jar lib in distributed cache. | Critical | . | Junping Du | Junping Du | +| [MAPREDUCE-6357](https://issues.apache.org/jira/browse/MAPREDUCE-6357) | MultipleOutputs.write() API should document that output committing is not utilized when input path is absolute | Major | documentation | Ivan Mitic | Dustin Cote | +| [YARN-3986](https://issues.apache.org/jira/browse/YARN-3986) | getTransferredContainers in AbstractYarnScheduler should be present in YarnScheduler interface instead | Major | scheduler | Varun Saxena | Varun Saxena | +| [HADOOP-12347](https://issues.apache.org/jira/browse/HADOOP-12347) | Fix mismatch parameter name in javadocs of AuthToken#setMaxInactives | Trivial | . | Xiaoyu Yao | Xiaoyu Yao | +| [HDFS-8942](https://issues.apache.org/jira/browse/HDFS-8942) | Update hyperlink to rack awareness page in HDFS Architecture documentation | Trivial | documentation | Masatake Iwasaki | Masatake Iwasaki | +| [YARN-3896](https://issues.apache.org/jira/browse/YARN-3896) | RMNode transitioned from RUNNING to REBOOTED because its response id had not been reset synchronously | Major | resourcemanager | Jun Gong | Jun Gong | +| [HDFS-8930](https://issues.apache.org/jira/browse/HDFS-8930) | Block report lease may leak if the 2nd full block report comes when NN is still in safemode | Minor | . | Jing Zhao | Jing Zhao | +| [HDFS-8932](https://issues.apache.org/jira/browse/HDFS-8932) | NPE thrown in NameNode when try to get "TotalSyncCount" metric before editLogStream initialization | Major | . | Surendra Singh Lilhore | Surendra Singh Lilhore | +| [HADOOP-12352](https://issues.apache.org/jira/browse/HADOOP-12352) | Delay in checkpointing Trash can leave trash for 2 intervals before deleting | Trivial | trash | Casey Brotherton | Casey Brotherton | +| [HDFS-8846](https://issues.apache.org/jira/browse/HDFS-8846) | Add a unit test for INotify functionality across a layout version upgrade | Major | namenode | Zhe Zhang | Zhe Zhang | +| [HDFS-8896](https://issues.apache.org/jira/browse/HDFS-8896) | DataNode object isn't GCed when shutdown, because it has GC root in ShutdownHookManager | Minor | test | Walter Su | Walter Su | +| [HDFS-8961](https://issues.apache.org/jira/browse/HDFS-8961) | Surpress findbug warnings of o.a.h.hdfs.shortcircuit.DfsClientShmManager.EndpointShmManager in hdfs-client | Major | . | Haohui Mai | Mingliang Liu | +| [HADOOP-12362](https://issues.apache.org/jira/browse/HADOOP-12362) | Set hadoop.tmp.dir and hadoop.log.dir in pom | Major | . | Charlie Helin | Charlie Helin | +| [HDFS-8969](https://issues.apache.org/jira/browse/HDFS-8969) | Clean up findbugs warnings for HDFS-8823 and HDFS-8932 | Major | namenode | Anu Engineer | Anu Engineer | +| [HDFS-8963](https://issues.apache.org/jira/browse/HDFS-8963) | Fix incorrect sign extension of xattr length in HDFS-8900 | Critical | . | Haohui Mai | Colin P. McCabe | +| [YARN-1556](https://issues.apache.org/jira/browse/YARN-1556) | NPE getting application report with a null appId | Minor | client | Steve Loughran | Weiwei Yang | +| [MAPREDUCE-6452](https://issues.apache.org/jira/browse/MAPREDUCE-6452) | NPE when intermediate encrypt enabled for LocalRunner | Major | . | Bibin A Chundatt | zhihai xu | +| [HDFS-8950](https://issues.apache.org/jira/browse/HDFS-8950) | NameNode refresh doesn't remove DataNodes that are no longer in the allowed list | Major | datanode, namenode | Daniel Templeton | Daniel Templeton | +| [HADOOP-12346](https://issues.apache.org/jira/browse/HADOOP-12346) | Increase some default timeouts / retries for S3a connector | Major | fs/s3 | Sean Mackrory | Sean Mackrory | +| [HADOOP-12359](https://issues.apache.org/jira/browse/HADOOP-12359) | hadoop fs -getmerge doc is wrong | Major | documentation | Daniel Templeton | Jagadesh Kiran N | +| [HADOOP-10365](https://issues.apache.org/jira/browse/HADOOP-10365) | BufferedOutputStream in FileUtil#unpackEntries() should be closed in finally block | Minor | util | Ted Yu | Kiran Kumar M R | +| [HDFS-8995](https://issues.apache.org/jira/browse/HDFS-8995) | Flaw in registration bookeeping can make DN die on reconnect | Critical | . | Kihwal Lee | Kihwal Lee | +| [HDFS-8388](https://issues.apache.org/jira/browse/HDFS-8388) | Time and Date format need to be in sync in NameNode UI page | Minor | . | Archana T | Surendra Singh Lilhore | +| [YARN-4073](https://issues.apache.org/jira/browse/YARN-4073) | Unused ApplicationACLsManager in ContainerManagerImpl | Trivial | . | Naganarasimha G R | Naganarasimha G R | +| [HDFS-9003](https://issues.apache.org/jira/browse/HDFS-9003) | ForkJoin thread pool leaks | Major | . | Kihwal Lee | Kihwal Lee | +| [HDFS-8885](https://issues.apache.org/jira/browse/HDFS-8885) | ByteRangeInputStream used in webhdfs does not override available() | Minor | webhdfs | Shradha Revankar | Shradha Revankar | +| [HADOOP-10318](https://issues.apache.org/jira/browse/HADOOP-10318) | Incorrect reference to nodeFile in RumenToSLSConverter error message | Minor | . | Ted Yu | Wei Yan | +| [HADOOP-12213](https://issues.apache.org/jira/browse/HADOOP-12213) | Interrupted exception can occur when Client#stop is called | Minor | . | Oleg Zhurakousky | Kuhu Shukla | +| [HDFS-9009](https://issues.apache.org/jira/browse/HDFS-9009) | Send metrics logs to NullAppender by default | Major | logging | Arpit Agarwal | Arpit Agarwal | +| [HDFS-8964](https://issues.apache.org/jira/browse/HDFS-8964) | When validating the edit log, do not read at or beyond the file offset that is being written | Major | journal-node, namenode | Zhe Zhang | Zhe Zhang | +| [HDFS-8939](https://issues.apache.org/jira/browse/HDFS-8939) | Test(S)WebHdfsFileContextMainOperations failing on branch-2 | Major | webhdfs | Jakob Homan | Chris Nauroth | +| [YARN-4103](https://issues.apache.org/jira/browse/YARN-4103) | RM WebServices missing scheme for appattempts logLinks | Major | . | Jonathan Eagles | Jonathan Eagles | +| [YARN-4105](https://issues.apache.org/jira/browse/YARN-4105) | Capacity Scheduler headroom for DRF is wrong | Major | capacityscheduler | Chang Li | Chang Li | +| [MAPREDUCE-6442](https://issues.apache.org/jira/browse/MAPREDUCE-6442) | Stack trace is missing when error occurs in client protocol provider's constructor | Major | client | Chang Li | Chang Li | +| [YARN-3591](https://issues.apache.org/jira/browse/YARN-3591) | Resource localization on a bad disk causes subsequent containers failure | Major | . | Lavkesh Lahngir | Lavkesh Lahngir | +| [YARN-4121](https://issues.apache.org/jira/browse/YARN-4121) | Typos in capacity scheduler documentation. | Trivial | documentation | Kai Sasaki | Kai Sasaki | +| [YARN-4096](https://issues.apache.org/jira/browse/YARN-4096) | App local logs are leaked if log aggregation fails to initialize for the app | Major | log-aggregation, nodemanager | Jason Lowe | Jason Lowe | +| [HADOOP-12388](https://issues.apache.org/jira/browse/HADOOP-12388) | Fix components' version information in the web page 'About the Cluster' | Minor | util | Jun Gong | Jun Gong | +| [HDFS-9033](https://issues.apache.org/jira/browse/HDFS-9033) | dfsadmin -metasave prints "NaN" for cache used% | Major | . | Archana T | Brahma Reddy Battula | +| [HDFS-8716](https://issues.apache.org/jira/browse/HDFS-8716) | introduce a new config specifically for safe mode block count | Major | . | Chang Li | Chang Li | +| [HDFS-8581](https://issues.apache.org/jira/browse/HDFS-8581) | ContentSummary on / skips further counts on yielding lock | Minor | namenode | tongshiquan | J.Andreina | +| [HDFS-6763](https://issues.apache.org/jira/browse/HDFS-6763) | Initialize file system-wide quota once on transitioning to active | Major | ha, namenode | Daryn Sharp | Kihwal Lee | +| [MAPREDUCE-6474](https://issues.apache.org/jira/browse/MAPREDUCE-6474) | ShuffleHandler can possibly exhaust nodemanager file descriptors | Major | mrv2, nodemanager | Nathan Roberts | Kuhu Shukla | +| [YARN-4106](https://issues.apache.org/jira/browse/YARN-4106) | NodeLabels for NM in distributed mode is not updated even after clusterNodelabel addition in RM | Major | . | Bibin A Chundatt | Bibin A Chundatt | +| [YARN-4115](https://issues.apache.org/jira/browse/YARN-4115) | Reduce loglevel of ContainerManagementProtocolProxy to Debug | Minor | . | Anubhav Dhoot | Anubhav Dhoot | +| [HADOOP-12348](https://issues.apache.org/jira/browse/HADOOP-12348) | MetricsSystemImpl creates MetricsSourceAdapter with wrong time unit parameter. | Major | metrics | zhihai xu | zhihai xu | +| [HDFS-9042](https://issues.apache.org/jira/browse/HDFS-9042) | Update document for the Storage policy name | Minor | documentation | J.Andreina | J.Andreina | +| [HDFS-9036](https://issues.apache.org/jira/browse/HDFS-9036) | In BlockPlacementPolicyWithNodeGroup#chooseLocalStorage , random node is selected eventhough fallbackToLocalRack is true. | Major | . | J.Andreina | J.Andreina | +| [HADOOP-12407](https://issues.apache.org/jira/browse/HADOOP-12407) | Test failing: hadoop.ipc.TestSaslRPC | Critical | security, test | Steve Loughran | Steve Loughran | +| [HADOOP-12087](https://issues.apache.org/jira/browse/HADOOP-12087) | [JDK8] Fix javadoc errors caused by incorrect or illegal tags | Major | documentation | Akira Ajisaka | Akira Ajisaka | +| [HDFS-9069](https://issues.apache.org/jira/browse/HDFS-9069) | TestNameNodeMetricsLogger failing -port in use | Critical | test | Steve Loughran | Steve Loughran | +| [HDFS-8996](https://issues.apache.org/jira/browse/HDFS-8996) | Consolidate validateLog and scanLog in FJM#EditLogFile | Major | journal-node, namenode | Zhe Zhang | Zhe Zhang | +| [YARN-4151](https://issues.apache.org/jira/browse/YARN-4151) | Fix findbugs errors in hadoop-yarn-server-common module | Major | . | MENG DING | MENG DING | +| [HDFS-9067](https://issues.apache.org/jira/browse/HDFS-9067) | o.a.h.hdfs.server.datanode.fsdataset.impl.TestLazyWriter is failing in trunk | Critical | . | Haohui Mai | Surendra Singh Lilhore | +| [MAPREDUCE-6472](https://issues.apache.org/jira/browse/MAPREDUCE-6472) | MapReduce AM should have java.io.tmpdir=./tmp to be consistent with tasks | Major | mr-am | Jason Lowe | Naganarasimha G R | +| [HADOOP-12374](https://issues.apache.org/jira/browse/HADOOP-12374) | Description of hdfs expunge command is confusing | Major | documentation, trash | Weiwei Yang | Weiwei Yang | +| [YARN-4078](https://issues.apache.org/jira/browse/YARN-4078) | getPendingResourceRequestForAttempt is present in AbstractYarnScheduler should be present in YarnScheduler interface instead | Minor | resourcemanager | Naganarasimha G R | Naganarasimha G R | +| [HDFS-9072](https://issues.apache.org/jira/browse/HDFS-9072) | Fix random failures in TestJMXGet | Critical | test | J.Andreina | J.Andreina | +| [HDFS-9073](https://issues.apache.org/jira/browse/HDFS-9073) | Fix failures in TestLazyPersistLockedMemory#testReleaseOnEviction | Critical | test | J.Andreina | J.Andreina | +| [YARN-3433](https://issues.apache.org/jira/browse/YARN-3433) | Jersey tests failing with Port in Use -again | Critical | build, test | Steve Loughran | Brahma Reddy Battula | +| [HADOOP-12417](https://issues.apache.org/jira/browse/HADOOP-12417) | TestWebDelegationToken failing with port in use | Major | test | Steve Loughran | Mingliang Liu | +| [MAPREDUCE-6481](https://issues.apache.org/jira/browse/MAPREDUCE-6481) | LineRecordReader may give incomplete record and wrong position/key information for uncompressed input sometimes. | Critical | mrv2 | zhihai xu | zhihai xu | +| [MAPREDUCE-5002](https://issues.apache.org/jira/browse/MAPREDUCE-5002) | AM could potentially allocate a reduce container to a map attempt | Major | mr-am | Jason Lowe | Chang Li | +| [MAPREDUCE-5982](https://issues.apache.org/jira/browse/MAPREDUCE-5982) | Task attempts that fail from the ASSIGNED state can disappear | Major | mr-am | Jason Lowe | Chang Li | +| [HADOOP-12386](https://issues.apache.org/jira/browse/HADOOP-12386) | RetryPolicies.RETRY\_FOREVER should be able to specify a retry interval | Major | . | Wangda Tan | Sunil G | +| [YARN-3697](https://issues.apache.org/jira/browse/YARN-3697) | FairScheduler: ContinuousSchedulingThread can fail to shutdown | Critical | fairscheduler | zhihai xu | zhihai xu | +| [HDFS-6955](https://issues.apache.org/jira/browse/HDFS-6955) | DN should reserve disk space for a full block when creating tmp files | Major | datanode | Arpit Agarwal | Kanaka Kumar Avvaru | +| [HDFS-5802](https://issues.apache.org/jira/browse/HDFS-5802) | NameNode does not check for inode type before traversing down a path | Trivial | namenode | Harsh J | Xiao Chen | +| [MAPREDUCE-6460](https://issues.apache.org/jira/browse/MAPREDUCE-6460) | TestRMContainerAllocator.testAttemptNotFoundCausesRMCommunicatorException fails | Major | test | zhihai xu | zhihai xu | +| [YARN-4167](https://issues.apache.org/jira/browse/YARN-4167) | NPE on RMActiveServices#serviceStop when store is null | Minor | . | Bibin A Chundatt | Bibin A Chundatt | +| [YARN-4113](https://issues.apache.org/jira/browse/YARN-4113) | RM should respect retry-interval when uses RetryPolicies.RETRY\_FOREVER | Critical | . | Wangda Tan | Sunil G | +| [YARN-4188](https://issues.apache.org/jira/browse/YARN-4188) | MoveApplicationAcrossQueuesResponse should be an abstract class | Minor | resourcemanager | Giovanni Matteo Fumarola | Giovanni Matteo Fumarola | +| [HDFS-9043](https://issues.apache.org/jira/browse/HDFS-9043) | Doc updation for commands in HDFS Federation | Minor | documentation | J.Andreina | J.Andreina | +| [HDFS-9013](https://issues.apache.org/jira/browse/HDFS-9013) | Deprecate NameNodeMXBean#getNNStarted in branch2 and remove from trunk | Major | namenode | Surendra Singh Lilhore | Surendra Singh Lilhore | +| [YARN-3975](https://issues.apache.org/jira/browse/YARN-3975) | WebAppProxyServlet should not redirect to RM page if AHS is enabled | Major | . | Mit Desai | Mit Desai | +| [HADOOP-12438](https://issues.apache.org/jira/browse/HADOOP-12438) | Reset RawLocalFileSystem.useDeprecatedFileStatus in TestLocalFileSystem | Trivial | test | Chris Nauroth | Chris Nauroth | +| [HDFS-9128](https://issues.apache.org/jira/browse/HDFS-9128) | TestWebHdfsFileContextMainOperations and TestSWebHdfsFileContextMainOperations fail due to invalid HDFS path on Windows. | Trivial | test | Chris Nauroth | Chris Nauroth | +| [YARN-4152](https://issues.apache.org/jira/browse/YARN-4152) | NM crash with NPE when LogAggregationService#stopContainer called for absent container | Critical | log-aggregation, nodemanager | Bibin A Chundatt | Bibin A Chundatt | +| [YARN-4044](https://issues.apache.org/jira/browse/YARN-4044) | Running applications information changes such as movequeue is not published to TimeLine server | Critical | resourcemanager, timelineserver | Sunil G | Sunil G | +| [HDFS-9076](https://issues.apache.org/jira/browse/HDFS-9076) | Log full path instead of inodeId in DFSClient#closeAllFilesBeingWritten() | Major | hdfs-client | Surendra Singh Lilhore | Surendra Singh Lilhore | +| [MAPREDUCE-6484](https://issues.apache.org/jira/browse/MAPREDUCE-6484) | Yarn Client uses local address instead of RM address as token renewer in a secure cluster when RM HA is enabled. | Major | client, security | zhihai xu | zhihai xu | +| [HADOOP-12437](https://issues.apache.org/jira/browse/HADOOP-12437) | Allow SecurityUtil to lookup alternate hostnames | Major | net, security | Arpit Agarwal | Arpit Agarwal | +| [HADOOP-12252](https://issues.apache.org/jira/browse/HADOOP-12252) | LocalDirAllocator should not throw NPE with empty string configuration. | Minor | fs | zhihai xu | zhihai xu | +| [YARN-3624](https://issues.apache.org/jira/browse/YARN-3624) | ApplicationHistoryServer reverses the order of the filters it gets | Major | timelineserver | Mit Desai | Mit Desai | +| [HDFS-9123](https://issues.apache.org/jira/browse/HDFS-9123) | Copying from the root to a subdirectory should be forbidden | Minor | fs | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [MAPREDUCE-6480](https://issues.apache.org/jira/browse/MAPREDUCE-6480) | archive-logs tool may miss applications | Major | . | Robert Kanter | Robert Kanter | +| [HDFS-9107](https://issues.apache.org/jira/browse/HDFS-9107) | Prevent NN's unrecoverable death spiral after full GC | Critical | namenode | Daryn Sharp | Daryn Sharp | +| [HDFS-9133](https://issues.apache.org/jira/browse/HDFS-9133) | ExternalBlockReader and ReplicaAccessor need to return -1 on read when at EOF | Major | hdfs-client | Colin P. McCabe | Colin P. McCabe | +| [HDFS-9080](https://issues.apache.org/jira/browse/HDFS-9080) | update htrace version to 4.0.1 | Major | tracing | Colin P. McCabe | Colin P. McCabe | +| [YARN-4204](https://issues.apache.org/jira/browse/YARN-4204) | ConcurrentModificationException in FairSchedulerQueueInfo | Major | . | Anubhav Dhoot | Anubhav Dhoot | +| [HDFS-9106](https://issues.apache.org/jira/browse/HDFS-9106) | Transfer failure during pipeline recovery causes permanent write failures | Critical | . | Kihwal Lee | Kihwal Lee | +| [HDFS-9147](https://issues.apache.org/jira/browse/HDFS-9147) | Fix the setting of visibleLength in ExternalBlockReader | Major | hdfs-client | Colin P. McCabe | Colin P. McCabe | +| [MAPREDUCE-6492](https://issues.apache.org/jira/browse/MAPREDUCE-6492) | AsyncDispatcher exit with NPE on TaskAttemptImpl#sendJHStartEventForAssignedFailTask | Critical | . | Bibin A Chundatt | Bibin A Chundatt | +| [HADOOP-12440](https://issues.apache.org/jira/browse/HADOOP-12440) | TestRPC#testRPCServerShutdown did not produce the desired thread states before shutting down | Minor | . | Xiao Chen | Xiao Chen | +| [HDFS-9092](https://issues.apache.org/jira/browse/HDFS-9092) | Nfs silently drops overlapping write requests and causes data copying to fail | Major | nfs | Yongjun Zhang | Yongjun Zhang | +| [YARN-4180](https://issues.apache.org/jira/browse/YARN-4180) | AMLauncher does not retry on failures when talking to NM | Critical | resourcemanager | Anubhav Dhoot | Anubhav Dhoot | +| [HDFS-9141](https://issues.apache.org/jira/browse/HDFS-9141) | Thread leak in Datanode#refreshVolumes | Major | datanode | Uma Maheswara Rao G | Uma Maheswara Rao G | +| [YARN-4066](https://issues.apache.org/jira/browse/YARN-4066) | Large number of queues choke fair scheduler | Major | fairscheduler | Johan Gustavsson | Johan Gustavsson | +| [HADOOP-12447](https://issues.apache.org/jira/browse/HADOOP-12447) | Clean up some htrace integration issues | Major | tracing | Colin P. McCabe | Colin P. McCabe | +| [HDFS-9174](https://issues.apache.org/jira/browse/HDFS-9174) | Fix findbugs warnings in FSOutputSummer.tracer and DirectoryScanner$ReportCompiler.currentThread | Critical | . | Yi Liu | Yi Liu | +| [HDFS-9001](https://issues.apache.org/jira/browse/HDFS-9001) | DFSUtil.getNsServiceRpcUris() can return too many entries in a non-HA, non-federated cluster | Major | . | Daniel Templeton | Daniel Templeton | +| [HADOOP-12448](https://issues.apache.org/jira/browse/HADOOP-12448) | TestTextCommand: use mkdirs rather than mkdir to create test directory | Major | test | Colin P. McCabe | Colin P. McCabe | +| [MAPREDUCE-6494](https://issues.apache.org/jira/browse/MAPREDUCE-6494) | Permission issue when running archive-logs tool as different users | Major | . | Robert Kanter | Robert Kanter | +| [MAPREDUCE-6497](https://issues.apache.org/jira/browse/MAPREDUCE-6497) | Fix wrong value of JOB\_FINISHED event in JobHistoryEventHandler | Major | . | Shinichi Yamashita | Shinichi Yamashita | +| [HADOOP-10296](https://issues.apache.org/jira/browse/HADOOP-10296) | Incorrect null check in SwiftRestClient#buildException() | Minor | . | Ted Yu | Kanaka Kumar Avvaru | +| [HADOOP-8437](https://issues.apache.org/jira/browse/HADOOP-8437) | getLocalPathForWrite should throw IOException for invalid paths | Major | fs | Brahma Reddy Battula | Brahma Reddy Battula | +| [MAPREDUCE-6485](https://issues.apache.org/jira/browse/MAPREDUCE-6485) | MR job hanged forever because all resources are taken up by reducers and the last map attempt never get resource to run | Critical | applicationmaster | Bob.zhao | Xianyin Xin | +| [HDFS-9191](https://issues.apache.org/jira/browse/HDFS-9191) | Typo in Hdfs.java. NoSuchElementException is misspelled | Trivial | documentation | Catherine Palmer | Catherine Palmer | +| [HDFS-9100](https://issues.apache.org/jira/browse/HDFS-9100) | HDFS Balancer does not respect dfs.client.use.datanode.hostname | Major | balancer & mover | Yongjun Zhang | Casey Brotherton | +| [YARN-3619](https://issues.apache.org/jira/browse/YARN-3619) | ContainerMetrics unregisters during getMetrics and leads to ConcurrentModificationException | Major | nodemanager | Jason Lowe | zhihai xu | +| [HDFS-9193](https://issues.apache.org/jira/browse/HDFS-9193) | Fix incorrect references the usages of the DN in dfshealth.js | Minor | . | Chang Li | Chang Li | +| [HADOOP-11098](https://issues.apache.org/jira/browse/HADOOP-11098) | [JDK8] Max Non Heap Memory default changed between JDK7 and 8 | Major | . | Travis Thompson | Tsuyoshi Ozawa | +| [HADOOP-12441](https://issues.apache.org/jira/browse/HADOOP-12441) | Fix kill command behavior under some Linux distributions. | Critical | . | Wangda Tan | Wangda Tan | +| [HADOOP-12452](https://issues.apache.org/jira/browse/HADOOP-12452) | Fix tracing documention reflecting the update to htrace-4 | Major | documentation | Masatake Iwasaki | Masatake Iwasaki | +| [YARN-4176](https://issues.apache.org/jira/browse/YARN-4176) | Resync NM nodelabels with RM periodically for distributed nodelabels | Major | . | Bibin A Chundatt | Bibin A Chundatt | +| [HDFS-7899](https://issues.apache.org/jira/browse/HDFS-7899) | Improve EOF error message | Minor | hdfs-client | Harsh J | Jagadesh Kiran N | +| [MAPREDUCE-6503](https://issues.apache.org/jira/browse/MAPREDUCE-6503) | archive-logs tool should use HADOOP\_PREFIX instead of HADOOP\_HOME | Major | . | Robert Kanter | Robert Kanter | +| [YARN-4209](https://issues.apache.org/jira/browse/YARN-4209) | RMStateStore FENCED state doesn't work due to updateFencedState called by stateMachine.doTransition | Critical | resourcemanager | zhihai xu | zhihai xu | +| [HDFS-9196](https://issues.apache.org/jira/browse/HDFS-9196) | Fix TestWebHdfsContentLength | Major | . | Tsuyoshi Ozawa | Masatake Iwasaki | +| [HDFS-9178](https://issues.apache.org/jira/browse/HDFS-9178) | Slow datanode I/O can cause a wrong node to be marked bad | Critical | . | Kihwal Lee | Kihwal Lee | +| [HADOOP-12465](https://issues.apache.org/jira/browse/HADOOP-12465) | Incorrect javadoc in WritableUtils.java | Minor | documentation | Martin Petricek | Jagadesh Kiran N | +| [HDFS-9176](https://issues.apache.org/jira/browse/HDFS-9176) | TestDirectoryScanner#testThrottling often fails. | Minor | test | Yi Liu | Daniel Templeton | +| [HDFS-9211](https://issues.apache.org/jira/browse/HDFS-9211) | Fix incorrect version in hadoop-hdfs-native-client/pom.xml from HDFS-9170 branch-2 backport | Major | build | Eric Payne | Eric Payne | +| [HDFS-9137](https://issues.apache.org/jira/browse/HDFS-9137) | DeadLock between DataNode#refreshVolumes and BPOfferService#registrationSucceeded | Major | datanode | Uma Maheswara Rao G | Uma Maheswara Rao G | +| [HDFS-8164](https://issues.apache.org/jira/browse/HDFS-8164) | cTime is 0 in VERSION file for newly formatted NameNode. | Minor | namenode | Chris Nauroth | Xiao Chen | +| [YARN-4235](https://issues.apache.org/jira/browse/YARN-4235) | FairScheduler PrimaryGroup does not handle empty groups returned for a user | Major | fairscheduler | Anubhav Dhoot | Anubhav Dhoot | +| [MAPREDUCE-6302](https://issues.apache.org/jira/browse/MAPREDUCE-6302) | Preempt reducers after a configurable timeout irrespective of headroom | Critical | . | mai shurong | Karthik Kambatla | +| [HDFS-9142](https://issues.apache.org/jira/browse/HDFS-9142) | Separating Configuration object for namenode(s) in MiniDFSCluster | Major | . | Siqi Li | Siqi Li | +| [HDFS-8941](https://issues.apache.org/jira/browse/HDFS-8941) | DistributedFileSystem listCorruptFileBlocks API should resolve relative path | Major | hdfs-client | Rakesh R | Rakesh R | +| [HDFS-9222](https://issues.apache.org/jira/browse/HDFS-9222) | Add hadoop-hdfs-client as a dependency of hadoop-hdfs-native-client | Major | . | Haohui Mai | Mingliang Liu | +| [YARN-4201](https://issues.apache.org/jira/browse/YARN-4201) | AMBlacklist does not work for minicluster | Major | resourcemanager | Jun Gong | Jun Gong | +| [HDFS-9215](https://issues.apache.org/jira/browse/HDFS-9215) | Suppress the RAT warnings in hdfs-native-client module | Minor | . | Haohui Mai | Haohui Mai | +| [HDFS-9224](https://issues.apache.org/jira/browse/HDFS-9224) | TestFileTruncate fails intermittently with BindException | Major | test | Brahma Reddy Battula | Brahma Reddy Battula | +| [YARN-4017](https://issues.apache.org/jira/browse/YARN-4017) | container-executor overuses PATH\_MAX | Major | nodemanager | Allen Wittenauer | Sidharta Seethana | +| [HDFS-8676](https://issues.apache.org/jira/browse/HDFS-8676) | Delayed rolling upgrade finalization can cause heartbeat expiration and write failures | Critical | . | Kihwal Lee | Walter Su | +| [HADOOP-12474](https://issues.apache.org/jira/browse/HADOOP-12474) | MiniKMS should use random ports for Jetty server by default | Major | . | Mingliang Liu | Mingliang Liu | +| [HADOOP-12449](https://issues.apache.org/jira/browse/HADOOP-12449) | TestDNS and TestNetUtils failing if no network | Minor | test | Steve Loughran | Steve Loughran | +| [HADOOP-11515](https://issues.apache.org/jira/browse/HADOOP-11515) | Upgrade jsch lib to jsch-0.1.51 to avoid problems running on java7 | Major | build | Johannes Zillmann | Tsuyoshi Ozawa | +| [HDFS-9235](https://issues.apache.org/jira/browse/HDFS-9235) | hdfs-native-client build getting errors when built with cmake 2.6 | Minor | hdfs-client | Eric Payne | Eric Payne | +| [HDFS-8779](https://issues.apache.org/jira/browse/HDFS-8779) | WebUI fails to display block IDs that are larger than 2^53 - 1 | Minor | webhdfs | Walter Su | Haohui Mai | +| [HDFS-9187](https://issues.apache.org/jira/browse/HDFS-9187) | Fix null pointer error in Globber when FS was not constructed via FileSystem#createFileSystem | Major | tracing | stack | Colin P. McCabe | +| [HDFS-1172](https://issues.apache.org/jira/browse/HDFS-1172) | Blocks in newly completed files are considered under-replicated too quickly | Major | namenode | Todd Lipcon | Masatake Iwasaki | +| [YARN-4250](https://issues.apache.org/jira/browse/YARN-4250) | NPE in AppSchedulingInfo#isRequestLabelChanged | Major | resourcemanager, scheduler | Brahma Reddy Battula | Brahma Reddy Battula | +| [HADOOP-12478](https://issues.apache.org/jira/browse/HADOOP-12478) | Shell.getWinUtilsPath() has been renamed Shell.getWinutilsPath() | Critical | util | Steve Loughran | Steve Loughran | +| [HDFS-9220](https://issues.apache.org/jira/browse/HDFS-9220) | Reading small file (\< 512 bytes) that is open for append fails due to incorrect checksum | Blocker | . | Bogdan Raducanu | Jing Zhao | +| [HADOOP-12479](https://issues.apache.org/jira/browse/HADOOP-12479) | ProtocMojo does not log the reason for a protoc compilation failure. | Minor | build | Chris Nauroth | Chris Nauroth | +| [HADOOP-11628](https://issues.apache.org/jira/browse/HADOOP-11628) | SPNEGO auth does not work with CNAMEs in JDK8 | Blocker | security | Daryn Sharp | Daryn Sharp | +| [YARN-2597](https://issues.apache.org/jira/browse/YARN-2597) | MiniYARNCluster should propagate reason for AHS not starting | Major | test | Steve Loughran | Steve Loughran | +| [YARN-4155](https://issues.apache.org/jira/browse/YARN-4155) | TestLogAggregationService.testLogAggregationServiceWithInterval failing | Critical | test | Steve Loughran | Bibin A Chundatt | +| [HADOOP-10941](https://issues.apache.org/jira/browse/HADOOP-10941) | Proxy user verification NPEs if remote host is unresolvable | Critical | ipc, security | Daryn Sharp | Benoy Antony | +| [HADOOP-12483](https://issues.apache.org/jira/browse/HADOOP-12483) | Maintain wrapped SASL ordering for postponed IPC responses | Critical | ipc | Daryn Sharp | Daryn Sharp | +| [HDFS-9237](https://issues.apache.org/jira/browse/HDFS-9237) | NPE at TestDataNodeVolumeFailureToleration#tearDown | Major | test | Brahma Reddy Battula | Brahma Reddy Battula | +| [HADOOP-12464](https://issues.apache.org/jira/browse/HADOOP-12464) | Interrupted client may try to fail-over and retry | Major | ipc | Kihwal Lee | Kihwal Lee | +| [YARN-4270](https://issues.apache.org/jira/browse/YARN-4270) | Limit application resource reservation on nodes for non-node/rack specific requests | Major | fairscheduler | Arun Suresh | Arun Suresh | +| [HDFS-9208](https://issues.apache.org/jira/browse/HDFS-9208) | Disabling atime may fail clients like distCp | Major | . | Kihwal Lee | Kihwal Lee | +| [HDFS-9270](https://issues.apache.org/jira/browse/HDFS-9270) | TestShortCircuitLocalRead should not leave socket after unit test | Minor | test | Masatake Iwasaki | Masatake Iwasaki | +| [HADOOP-12418](https://issues.apache.org/jira/browse/HADOOP-12418) | TestRPC.testRPCInterruptedSimple fails intermittently | Major | test | Steve Loughran | Kihwal Lee | +| [MAPREDUCE-6495](https://issues.apache.org/jira/browse/MAPREDUCE-6495) | Docs for archive-logs tool | Major | documentation | Robert Kanter | Robert Kanter | +| [HDFS-3059](https://issues.apache.org/jira/browse/HDFS-3059) | ssl-server.xml causes NullPointer | Minor | datanode, security | Evert Lammerts | Xiao Chen | +| [HDFS-9274](https://issues.apache.org/jira/browse/HDFS-9274) | Default value of dfs.datanode.directoryscan.throttle.limit.ms.per.sec should be consistent | Trivial | datanode | Yi Liu | Yi Liu | +| [MAPREDUCE-6518](https://issues.apache.org/jira/browse/MAPREDUCE-6518) | Set SO\_KEEPALIVE on shuffle connections | Major | mrv2, nodemanager | Nathan Roberts | Chang Li | +| [HDFS-9225](https://issues.apache.org/jira/browse/HDFS-9225) | Fix intermittent test failure of TestBlockManager.testBlocksAreNotUnderreplicatedInSingleRack | Minor | test | Masatake Iwasaki | Masatake Iwasaki | +| [HDFS-9273](https://issues.apache.org/jira/browse/HDFS-9273) | ACLs on root directory may be lost after NN restart | Critical | namenode | Xiao Chen | Xiao Chen | +| [YARN-4000](https://issues.apache.org/jira/browse/YARN-4000) | RM crashes with NPE if leaf queue becomes parent queue during restart | Major | capacityscheduler, resourcemanager | Jason Lowe | Varun Saxena | +| [HADOOP-9692](https://issues.apache.org/jira/browse/HADOOP-9692) | Improving log message when SequenceFile reader throws EOFException on zero-length file | Major | . | Chu Tong | Zhe Zhang | +| [YARN-4256](https://issues.apache.org/jira/browse/YARN-4256) | YARN fair scheduler vcores with decimal values | Minor | fairscheduler | Prabhu Joseph | Jun Gong | +| [HADOOP-12484](https://issues.apache.org/jira/browse/HADOOP-12484) | Single File Rename Throws Incorrectly In Potential Race Condition Scenarios | Major | tools | Gaurav Kanade | Gaurav Kanade | +| [HDFS-9286](https://issues.apache.org/jira/browse/HDFS-9286) | HttpFs does not parse ACL syntax correctly for operation REMOVEACLENTRIES | Major | fs | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [YARN-4009](https://issues.apache.org/jira/browse/YARN-4009) | CORS support for ResourceManager REST API | Major | . | Prakash Ramachandran | Varun Vasudev | +| [YARN-4041](https://issues.apache.org/jira/browse/YARN-4041) | Slow delegation token renewal can severely prolong RM recovery | Major | resourcemanager | Jason Lowe | Sunil G | +| [HDFS-9290](https://issues.apache.org/jira/browse/HDFS-9290) | DFSClient#callAppend() is not backward compatible for slightly older NameNodes | Blocker | . | Tony Wu | Tony Wu | +| [HDFS-9301](https://issues.apache.org/jira/browse/HDFS-9301) | HDFS clients can't construct HdfsConfiguration instances | Major | . | Steve Loughran | Mingliang Liu | +| [YARN-4294](https://issues.apache.org/jira/browse/YARN-4294) | [JDK8] Fix javadoc errors caused by wrong reference and illegal tag | Blocker | build, documentation | Akira Ajisaka | Akira Ajisaka | +| [YARN-4289](https://issues.apache.org/jira/browse/YARN-4289) | TestDistributedShell failing with bind exception | Major | test | Brahma Reddy Battula | Brahma Reddy Battula | +| [YARN-4296](https://issues.apache.org/jira/browse/YARN-4296) | DistributedShell Log.info is not friendly | Major | applications/distributed-shell | Xiaowei Wang | Xiaowei Wang | +| [YARN-4246](https://issues.apache.org/jira/browse/YARN-4246) | NPE while listing app attempt | Major | . | Varun Saxena | nijel | +| [YARN-4223](https://issues.apache.org/jira/browse/YARN-4223) | Findbugs warnings in hadoop-yarn-server-nodemanager project | Minor | nodemanager | Varun Saxena | Varun Saxena | +| [HADOOP-12513](https://issues.apache.org/jira/browse/HADOOP-12513) | Dockerfile lacks initial 'apt-get update' | Trivial | build | Akihiro Suda | Akihiro Suda | +| [YARN-4284](https://issues.apache.org/jira/browse/YARN-4284) | condition for AM blacklisting is too narrow | Major | resourcemanager | Sangjin Lee | Sangjin Lee | +| [HDFS-9268](https://issues.apache.org/jira/browse/HDFS-9268) | fuse\_dfs chown crashes when uid is passed as -1 | Minor | . | Wei-Chiu Chuang | Colin P. McCabe | +| [HDFS-9284](https://issues.apache.org/jira/browse/HDFS-9284) | fsck command should not print exception trace when file not found | Major | . | Jagadesh Kiran N | Jagadesh Kiran N | +| [HDFS-9305](https://issues.apache.org/jira/browse/HDFS-9305) | Delayed heartbeat processing causes storm of subsequent heartbeats | Major | datanode | Chris Nauroth | Arpit Agarwal | +| [YARN-4169](https://issues.apache.org/jira/browse/YARN-4169) | Fix racing condition of TestNodeStatusUpdaterForLabels | Critical | test | Steve Loughran | Naganarasimha G R | +| [YARN-4300](https://issues.apache.org/jira/browse/YARN-4300) | [JDK8] Fix javadoc errors caused by wrong tags | Blocker | build, documentation | Akira Ajisaka | Akira Ajisaka | +| [YARN-4302](https://issues.apache.org/jira/browse/YARN-4302) | SLS not able start due to NPE in SchedulerApplicationAttempt#getResourceUsageReport | Major | . | Bibin A Chundatt | Bibin A Chundatt | +| [HADOOP-12178](https://issues.apache.org/jira/browse/HADOOP-12178) | NPE during handling of SASL setup if problem with SASL resolver class | Minor | ipc | Steve Loughran | Steve Loughran | +| [HADOOP-11685](https://issues.apache.org/jira/browse/HADOOP-11685) | StorageException complaining " no lease ID" during HBase distributed log splitting | Major | tools | Duo Xu | Duo Xu | +| [HDFS-9231](https://issues.apache.org/jira/browse/HDFS-9231) | fsck doesn't list correct file path when Bad Replicas/Blocks are in a snapshot | Major | snapshots | Xiao Chen | Xiao Chen | +| [HDFS-9302](https://issues.apache.org/jira/browse/HDFS-9302) | WebHDFS truncate throws NullPointerException if newLength is not provided | Minor | webhdfs | Karthik Palaniappan | Jagadesh Kiran N | +| [YARN-4251](https://issues.apache.org/jira/browse/YARN-4251) | TestAMRMClientOnRMRestart#testAMRMClientOnAMRMTokenRollOverOnRMRestart is failing | Major | test | Brahma Reddy Battula | Brahma Reddy Battula | +| [HADOOP-12519](https://issues.apache.org/jira/browse/HADOOP-12519) | hadoop-azure tests should avoid creating a metrics configuration file in the module root directory. | Minor | fs/azure, test | Chris Nauroth | Chris Nauroth | +| [HDFS-9279](https://issues.apache.org/jira/browse/HDFS-9279) | Decomissioned capacity should not be considered for configured/used capacity | Major | . | Kuhu Shukla | Kuhu Shukla | +| [HDFS-9044](https://issues.apache.org/jira/browse/HDFS-9044) | Give Priority to FavouredNodes , before selecting nodes from FavouredNode's Node Group | Major | . | J.Andreina | J.Andreina | +| [YARN-4130](https://issues.apache.org/jira/browse/YARN-4130) | Duplicate declaration of ApplicationId in RMAppManager#submitApplication method | Trivial | resourcemanager | Kai Sasaki | Kai Sasaki | +| [YARN-4288](https://issues.apache.org/jira/browse/YARN-4288) | NodeManager restart should keep retrying to register to RM while connection exception happens during RM failed over. | Critical | nodemanager | Junping Du | Junping Du | +| [MAPREDUCE-6515](https://issues.apache.org/jira/browse/MAPREDUCE-6515) | Update Application priority in AM side from AM-RM heartbeat | Major | applicationmaster | Sunil G | Sunil G | +| [HDFS-9332](https://issues.apache.org/jira/browse/HDFS-9332) | Fix Precondition failures from NameNodeEditLogRoller while saving namespace | Major | . | Andrew Wang | Andrew Wang | +| [YARN-4313](https://issues.apache.org/jira/browse/YARN-4313) | Race condition in MiniMRYarnCluster when getting history server address | Major | . | Jian He | Jian He | +| [YARN-4312](https://issues.apache.org/jira/browse/YARN-4312) | TestSubmitApplicationWithRMHA fails on branch-2.7 and branch-2.6 as some of the test cases time out | Major | . | Varun Saxena | Varun Saxena | +| [YARN-4320](https://issues.apache.org/jira/browse/YARN-4320) | TestJobHistoryEventHandler fails as AHS in MiniYarnCluster no longer binds to default port 8188 | Major | . | Varun Saxena | Varun Saxena | +| [MAPREDUCE-6528](https://issues.apache.org/jira/browse/MAPREDUCE-6528) | Memory leak for HistoryFileManager.getJobSummary() | Critical | jobhistoryserver | Junping Du | Junping Du | +| [MAPREDUCE-6451](https://issues.apache.org/jira/browse/MAPREDUCE-6451) | DistCp has incorrect chunkFilePath for multiple jobs when strategy is dynamic | Major | distcp | Kuhu Shukla | Kuhu Shukla | +| [HADOOP-12533](https://issues.apache.org/jira/browse/HADOOP-12533) | Introduce FileNotFoundException in WASB for read and seek API | Major | tools | Dushyanth | Dushyanth | +| [HADOOP-12508](https://issues.apache.org/jira/browse/HADOOP-12508) | delete fails with exception when lease is held on blob | Blocker | fs/azure | Gaurav Kanade | Gaurav Kanade | +| [HDFS-9329](https://issues.apache.org/jira/browse/HDFS-9329) | TestBootstrapStandby#testRateThrottling is flaky because fsimage size is smaller than IO buffer size | Minor | test | Zhe Zhang | Zhe Zhang | +| [HDFS-9313](https://issues.apache.org/jira/browse/HDFS-9313) | Possible NullPointerException in BlockManager if no excess replica can be chosen | Major | . | Ming Ma | Ming Ma | +| [HADOOP-12542](https://issues.apache.org/jira/browse/HADOOP-12542) | TestDNS fails on Windows after HADOOP-12437. | Major | net | Chris Nauroth | Chris Nauroth | +| [YARN-4326](https://issues.apache.org/jira/browse/YARN-4326) | Fix TestDistributedShell timeout as AHS in MiniYarnCluster no longer binds to default port 8188 | Major | . | MENG DING | MENG DING | +| [HDFS-9289](https://issues.apache.org/jira/browse/HDFS-9289) | Make DataStreamer#block thread safe and verify genStamp in commitBlock | Critical | . | Chang Li | Chang Li | +| [YARN-4127](https://issues.apache.org/jira/browse/YARN-4127) | RM fail with noAuth error if switched from failover mode to non-failover mode | Major | resourcemanager | Jian He | Varun Saxena | +| [HDFS-9351](https://issues.apache.org/jira/browse/HDFS-9351) | checkNNStartup() need to be called when fsck calls FSNamesystem.getSnapshottableDirs() | Major | namenode | Yongjun Zhang | Xiao Chen | +| [HADOOP-12296](https://issues.apache.org/jira/browse/HADOOP-12296) | when setnetgrent returns 0 in linux, exception should be thrown | Major | . | Chang Li | Chang Li | +| [HDFS-9357](https://issues.apache.org/jira/browse/HDFS-9357) | NN UI renders icons of decommissioned DN incorrectly | Critical | . | Archana T | Surendra Singh Lilhore | +| [HADOOP-12540](https://issues.apache.org/jira/browse/HADOOP-12540) | TestAzureFileSystemInstrumentation#testClientErrorMetrics fails intermittently due to assumption that a lease error will be thrown. | Major | fs/azure, test | Chris Nauroth | Gaurav Kanade | +| [HDFS-9360](https://issues.apache.org/jira/browse/HDFS-9360) | Storage type usage isn't updated properly after file deletion | Major | . | Ming Ma | Ming Ma | +| [HDFS-9378](https://issues.apache.org/jira/browse/HDFS-9378) | hadoop-hdfs-client tests do not write logs. | Minor | test | Chris Nauroth | Chris Nauroth | +| [HDFS-4937](https://issues.apache.org/jira/browse/HDFS-4937) | ReplicationMonitor can infinite-loop in BlockPlacementPolicyDefault#chooseRandom() | Major | namenode | Kihwal Lee | Kihwal Lee | +| [HDFS-9372](https://issues.apache.org/jira/browse/HDFS-9372) | Remove dead code in DataStorage.recoverTransitionRead | Major | datanode | Duo Zhang | Duo Zhang | +| [HDFS-9384](https://issues.apache.org/jira/browse/HDFS-9384) | TestWebHdfsContentLength intermittently hangs and fails due to TCP conversation mismatch between client and server. | Minor | test | Chris Nauroth | Chris Nauroth | +| [HDFS-6481](https://issues.apache.org/jira/browse/HDFS-6481) | DatanodeManager#getDatanodeStorageInfos() should check the length of storageIDs | Minor | namenode | Ted Yu | Tsz Wo Nicholas Sze | +| [HDFS-9318](https://issues.apache.org/jira/browse/HDFS-9318) | considerLoad factor can be improved | Major | . | Kuhu Shukla | Kuhu Shukla | +| [HDFS-9236](https://issues.apache.org/jira/browse/HDFS-9236) | Missing sanity check for block size during block recovery | Major | datanode | Tony Wu | Tony Wu | +| [HDFS-9394](https://issues.apache.org/jira/browse/HDFS-9394) | branch-2 hadoop-hdfs-client fails during FileSystem ServiceLoader initialization, because HftpFileSystem is missing. | Critical | hdfs-client | Chris Nauroth | Mingliang Liu | +| [HADOOP-12526](https://issues.apache.org/jira/browse/HADOOP-12526) | [Branch-2] there are duplicate dependency definitions in pom's | Major | build | Sangjin Lee | Sangjin Lee | +| [MAPREDUCE-5763](https://issues.apache.org/jira/browse/MAPREDUCE-5763) | Warn message about httpshuffle in NM logs | Major | . | Sandy Ryza | Akira Ajisaka | +| [HDFS-9383](https://issues.apache.org/jira/browse/HDFS-9383) | TestByteArrayManager#testByteArrayManager fails | Major | . | Kihwal Lee | Tsz Wo Nicholas Sze | +| [HDFS-9249](https://issues.apache.org/jira/browse/HDFS-9249) | NPE is thrown if an IOException is thrown in NameNode constructor | Minor | namenode | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HDFS-2261](https://issues.apache.org/jira/browse/HDFS-2261) | AOP unit tests are not getting compiled or run | Minor | test | Giridharan Kesavan | Haohui Mai | +| [HDFS-9401](https://issues.apache.org/jira/browse/HDFS-9401) | Fix findbugs warnings in BlockRecoveryWorker | Major | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [HADOOP-12482](https://issues.apache.org/jira/browse/HADOOP-12482) | Race condition in JMX cache update | Major | . | Tony Wu | Tony Wu | +| [HDFS-9364](https://issues.apache.org/jira/browse/HDFS-9364) | Unnecessary DNS resolution attempts when creating NameNodeProxies | Major | ha, performance | Xiao Chen | Xiao Chen | +| [HADOOP-12560](https://issues.apache.org/jira/browse/HADOOP-12560) | Fix sprintf warnings in {{DomainSocket.c}} introduced by HADOOP-12344 | Major | native | Colin P. McCabe | Mingliang Liu | +| [HDFS-9245](https://issues.apache.org/jira/browse/HDFS-9245) | Fix findbugs warnings in hdfs-nfs/WriteCtx | Major | nfs | Mingliang Liu | Mingliang Liu | +| [YARN-4241](https://issues.apache.org/jira/browse/YARN-4241) | Fix typo of property name in yarn-default.xml | Major | documentation | Anthony Rojas | Anthony Rojas | +| [MAPREDUCE-6533](https://issues.apache.org/jira/browse/MAPREDUCE-6533) | testDetermineCacheVisibilities of TestClientDistributedCacheManager is broken | Major | . | Chang Li | Chang Li | +| [HDFS-9396](https://issues.apache.org/jira/browse/HDFS-9396) | Total files and directories on jmx and web UI on standby is uninitialized | Blocker | . | Kihwal Lee | Kihwal Lee | +| [MAPREDUCE-6540](https://issues.apache.org/jira/browse/MAPREDUCE-6540) | TestMRTimelineEventHandling fails | Major | test | Sangjin Lee | Sangjin Lee | +| [YARN-4347](https://issues.apache.org/jira/browse/YARN-4347) | Resource manager fails with Null pointer exception | Major | yarn | Yesha Vora | Jian He | +| [HADOOP-12545](https://issues.apache.org/jira/browse/HADOOP-12545) | Hadoop javadoc has broken links for AccessControlList, ImpersonationProvider, DefaultImpersonationProvider, and DistCp | Major | documentation | Mohammad Arshad | Mohammad Arshad | +| [YARN-4354](https://issues.apache.org/jira/browse/YARN-4354) | Public resource localization fails with NPE | Blocker | nodemanager | Jason Lowe | Jason Lowe | +| [HDFS-9413](https://issues.apache.org/jira/browse/HDFS-9413) | getContentSummary() on standby should throw StandbyException | Critical | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [HDFS-9358](https://issues.apache.org/jira/browse/HDFS-9358) | TestNodeCount#testNodeCount timed out | Major | test | Wei-Chiu Chuang | Masatake Iwasaki | +| [HDFS-9397](https://issues.apache.org/jira/browse/HDFS-9397) | Fix typo for readChecksum() LOG.warn in BlockSender.java | Trivial | . | Enrique Flores | Enrique Flores | +| [HDFS-9400](https://issues.apache.org/jira/browse/HDFS-9400) | TestRollingUpgradeRollback fails on branch-2. | Blocker | . | Chris Nauroth | Brahma Reddy Battula | +| [YARN-4367](https://issues.apache.org/jira/browse/YARN-4367) | SLS webapp doesn't load | Major | scheduler-load-simulator | Karthik Kambatla | Karthik Kambatla | +| [HDFS-9431](https://issues.apache.org/jira/browse/HDFS-9431) | DistributedFileSystem#concat fails if the target path is relative. | Major | hdfs-client | Kazuho Fujii | Kazuho Fujii | +| [YARN-2859](https://issues.apache.org/jira/browse/YARN-2859) | ApplicationHistoryServer binds to default port 8188 in MiniYARNCluster | Critical | timelineserver | Hitesh Shah | Vinod Kumar Vavilapalli | +| [YARN-4374](https://issues.apache.org/jira/browse/YARN-4374) | RM capacity scheduler UI rounds user limit factor | Major | capacityscheduler | Chang Li | Chang Li | +| [YARN-3769](https://issues.apache.org/jira/browse/YARN-3769) | Consider user limit when calculating total pending resource for preemption policy in Capacity Scheduler | Major | capacityscheduler | Eric Payne | Eric Payne | +| [HDFS-9443](https://issues.apache.org/jira/browse/HDFS-9443) | Disabling HDFS client socket cache causes logging message printed to console for CLI commands. | Trivial | hdfs-client | Chris Nauroth | Chris Nauroth | +| [HADOOP-11218](https://issues.apache.org/jira/browse/HADOOP-11218) | Add TLSv1.1,TLSv1.2 to KMS, HttpFS, SSLFactory | Critical | kms | Robert Kanter | Vijay Singh | +| [HADOOP-12467](https://issues.apache.org/jira/browse/HADOOP-12467) | Respect user-defined JAVA\_LIBRARY\_PATH in Windows Hadoop scripts | Minor | scripts | Radhey Shah | Radhey Shah | +| [HADOOP-12181](https://issues.apache.org/jira/browse/HADOOP-12181) | Fix intermittent test failure of TestZKSignerSecretProvider | Minor | . | Masatake Iwasaki | Masatake Iwasaki | +| [HDFS-6885](https://issues.apache.org/jira/browse/HDFS-6885) | Fix wrong use of BytesWritable in FSEditLogOp#RenameOp | Minor | namenode | Yi Liu | Yi Liu | +| [HADOOP-12098](https://issues.apache.org/jira/browse/HADOOP-12098) | Remove redundant test dependencies in Hadoop Archives | Minor | . | Varun Saxena | Varun Saxena | +| [HDFS-7897](https://issues.apache.org/jira/browse/HDFS-7897) | Shutdown metrics when stopping JournalNode | Major | . | zhouyingchao | zhouyingchao | +| [HADOOP-11149](https://issues.apache.org/jira/browse/HADOOP-11149) | Increase the timeout of TestZKFailoverController | Major | test | Rajat Jain | Steve Loughran | +| [HADOOP-11677](https://issues.apache.org/jira/browse/HADOOP-11677) | Add cookie flags for logs and static contexts | Major | . | nijel | nijel | +| [HDFS-9356](https://issues.apache.org/jira/browse/HDFS-9356) | Decommissioning node does not have Last Contact value in the UI | Major | . | Archana T | Surendra Singh Lilhore | +| [HADOOP-12313](https://issues.apache.org/jira/browse/HADOOP-12313) | NPE in JvmPauseMonitor when calling stop() before start() | Critical | . | Rohith Sharma K S | Gabor Liptak | +| [HDFS-9428](https://issues.apache.org/jira/browse/HDFS-9428) | Fix intermittent failure of TestDNFencing.testQueueingWithAppend | Minor | test | Masatake Iwasaki | Masatake Iwasaki | +| [HDFS-9435](https://issues.apache.org/jira/browse/HDFS-9435) | TestBlockRecovery#testRBWReplicas is failing intermittently | Major | . | Rakesh R | Rakesh R | +| [HADOOP-12577](https://issues.apache.org/jira/browse/HADOOP-12577) | Bump up commons-collections version to 3.2.2 to address a security flaw | Blocker | build, security | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [YARN-4344](https://issues.apache.org/jira/browse/YARN-4344) | NMs reconnecting with changed capabilities can lead to wrong cluster resource calculations | Critical | resourcemanager | Varun Vasudev | Varun Vasudev | +| [HADOOP-9822](https://issues.apache.org/jira/browse/HADOOP-9822) | create constant MAX\_CAPACITY in RetryCache rather than hard-coding 16 in RetryCache constructor | Minor | . | Tsuyoshi Ozawa | Tsuyoshi Ozawa | +| [YARN-4298](https://issues.apache.org/jira/browse/YARN-4298) | Fix findbugs warnings in hadoop-yarn-common | Minor | . | Varun Saxena | Sunil G | +| [YARN-4387](https://issues.apache.org/jira/browse/YARN-4387) | Fix typo in FairScheduler log message | Minor | fairscheduler | Xin Wang | Xin Wang | +| [HDFS-6101](https://issues.apache.org/jira/browse/HDFS-6101) | TestReplaceDatanodeOnFailure fails occasionally | Major | test | Arpit Agarwal | Wei-Chiu Chuang | +| [HDFS-8855](https://issues.apache.org/jira/browse/HDFS-8855) | Webhdfs client leaks active NameNode connections | Major | webhdfs | Bob Hansen | Xiaobing Zhou | +| [HDFS-8335](https://issues.apache.org/jira/browse/HDFS-8335) | FSNamesystem should construct FSPermissionChecker only if permission is enabled | Major | . | David Bryson | Gabor Liptak | +| [MAPREDUCE-5883](https://issues.apache.org/jira/browse/MAPREDUCE-5883) | "Total megabyte-seconds" in job counters is slightly misleading | Minor | . | Nathan Roberts | Nathan Roberts | +| [YARN-4365](https://issues.apache.org/jira/browse/YARN-4365) | FileSystemNodeLabelStore should check for root dir existence on startup | Major | resourcemanager | Jason Lowe | Kuhu Shukla | +| [HDFS-8807](https://issues.apache.org/jira/browse/HDFS-8807) | dfs.datanode.data.dir does not handle spaces between storageType and URI correctly | Major | datanode | Anu Engineer | Anu Engineer | +| [HADOOP-12415](https://issues.apache.org/jira/browse/HADOOP-12415) | hdfs and nfs builds broken on -missing compile-time dependency on netty | Major | nfs | Konstantin Boudnik | Tom Zeng | +| [MAPREDUCE-6553](https://issues.apache.org/jira/browse/MAPREDUCE-6553) | Replace '\u2b05' with '\<-' in rendering job configuration | Minor | jobhistoryserver | Akira Ajisaka | Gabor Liptak | +| [HADOOP-12598](https://issues.apache.org/jira/browse/HADOOP-12598) | add XML namespace declarations for some hadoop/tools modules | Minor | build, tools | Xin Wang | Xin Wang | +| [YARN-4380](https://issues.apache.org/jira/browse/YARN-4380) | TestResourceLocalizationService.testDownloadingResourcesOnContainerKill fails intermittently | Major | test | Tsuyoshi Ozawa | Varun Saxena | +| [MAPREDUCE-6557](https://issues.apache.org/jira/browse/MAPREDUCE-6557) | Some tests in mapreduce-client-app are writing outside of target | Blocker | build | Allen Wittenauer | Akira Ajisaka | +| [HDFS-9459](https://issues.apache.org/jira/browse/HDFS-9459) | hadoop-hdfs-native-client fails test build on Windows after transition to ctest. | Blocker | build, test | Chris Nauroth | Chris Nauroth | +| [HDFS-9407](https://issues.apache.org/jira/browse/HDFS-9407) | TestFileTruncate fails with BindException | Major | test | Brahma Reddy Battula | Brahma Reddy Battula | +| [HDFS-9467](https://issues.apache.org/jira/browse/HDFS-9467) | Fix data race accessing writeLockHeldTimeStamp in FSNamesystem | Major | namenode | Mingliang Liu | Mingliang Liu | +| [MAPREDUCE-6549](https://issues.apache.org/jira/browse/MAPREDUCE-6549) | multibyte delimiters with LineRecordReader cause duplicate records | Major | mrv1, mrv2 | Dustin Cote | Wilfred Spiegelenburg | +| [MAPREDUCE-6550](https://issues.apache.org/jira/browse/MAPREDUCE-6550) | archive-logs tool changes log ownership to the Yarn user when using DefaultContainerExecutor | Major | . | Robert Kanter | Robert Kanter | +| [HADOOP-12468](https://issues.apache.org/jira/browse/HADOOP-12468) | Partial group resolution failure should not result in user lockout | Minor | security | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HDFS-9426](https://issues.apache.org/jira/browse/HDFS-9426) | Rollingupgrade finalization is not backward compatible | Blocker | . | Kihwal Lee | Kihwal Lee | +| [HADOOP-10406](https://issues.apache.org/jira/browse/HADOOP-10406) | TestIPC.testIpcWithReaderQueuing may fail | Major | ipc | Tsz Wo Nicholas Sze | Xiao Chen | +| [HDFS-9470](https://issues.apache.org/jira/browse/HDFS-9470) | Encryption zone on root not loaded from fsimage after NN restart | Critical | . | Xiao Chen | Xiao Chen | +| [HDFS-9336](https://issues.apache.org/jira/browse/HDFS-9336) | deleteSnapshot throws NPE when snapshotname is null | Major | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [HADOOP-12609](https://issues.apache.org/jira/browse/HADOOP-12609) | Fix intermittent failure of TestDecayRpcScheduler | Minor | ipc, test | Masatake Iwasaki | Masatake Iwasaki | +| [HDFS-6533](https://issues.apache.org/jira/browse/HDFS-6533) | TestBPOfferService#testBasicFunctionalitytest fails intermittently | Major | datanode, hdfs-client | Yongjun Zhang | Wei-Chiu Chuang | +| [YARN-4398](https://issues.apache.org/jira/browse/YARN-4398) | Yarn recover functionality causes the cluster running slowly and the cluster usage rate is far below 100 | Major | resourcemanager | NING DING | NING DING | +| [HDFS-9294](https://issues.apache.org/jira/browse/HDFS-9294) | DFSClient deadlock when close file and failed to renew lease | Blocker | hdfs-client | DENG FEI | Brahma Reddy Battula | +| [HADOOP-12565](https://issues.apache.org/jira/browse/HADOOP-12565) | Replace DSA with RSA for SSH key type in SingleCluster.md | Minor | documentation | Alexander Veit | Mingliang Liu | +| [YARN-4408](https://issues.apache.org/jira/browse/YARN-4408) | NodeManager still reports negative running containers | Major | nodemanager | Robert Kanter | Robert Kanter | +| [HDFS-9430](https://issues.apache.org/jira/browse/HDFS-9430) | Remove waitForLoadingFSImage since checkNNStartup has ensured image loaded and namenode started. | Major | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [YARN-4392](https://issues.apache.org/jira/browse/YARN-4392) | ApplicationCreatedEvent event time resets after RM restart/failover | Critical | . | Xuan Gong | Naganarasimha G R | +| [YARN-4422](https://issues.apache.org/jira/browse/YARN-4422) | Generic AHS sometimes doesn't show started, node, or logs on App page | Major | . | Eric Payne | Eric Payne | +| [YARN-4424](https://issues.apache.org/jira/browse/YARN-4424) | Fix deadlock in RMAppImpl | Blocker | . | Yesha Vora | Jian He | +| [HADOOP-12617](https://issues.apache.org/jira/browse/HADOOP-12617) | SPNEGO authentication request to non-default realm gets default realm name inserted in target server principal | Major | security | Matt Foley | Matt Foley | +| [YARN-4431](https://issues.apache.org/jira/browse/YARN-4431) | Not necessary to do unRegisterNM() if NM get stop due to failed to connect to RM | Major | nodemanager | Junping Du | Junping Du | +| [YARN-4421](https://issues.apache.org/jira/browse/YARN-4421) | Remove dead code in RmAppImpl.RMAppRecoveredTransition | Trivial | resourcemanager | Daniel Templeton | Daniel Templeton | +| [HADOOP-12618](https://issues.apache.org/jira/browse/HADOOP-12618) | NPE in TestSequenceFile | Major | test | Brahma Reddy Battula | Brahma Reddy Battula | +| [YARN-4434](https://issues.apache.org/jira/browse/YARN-4434) | NodeManager Disk Checker parameter documentation is not correct | Minor | documentation, nodemanager | Takashi Ohnishi | Weiwei Yang | +| [HADOOP-12602](https://issues.apache.org/jira/browse/HADOOP-12602) | TestMetricsSystemImpl#testQSize occasionally fail | Major | test | Wei-Chiu Chuang | Masatake Iwasaki | +| [HDFS-9519](https://issues.apache.org/jira/browse/HDFS-9519) | Some coding improvement in SecondaryNameNode#main | Major | namenode | Yongjun Zhang | Xiao Chen | +| [HDFS-9514](https://issues.apache.org/jira/browse/HDFS-9514) | TestDistributedFileSystem.testDFSClientPeerWriteTimeout failing; exception being swallowed | Major | hdfs-client, test | Steve Loughran | Wei-Chiu Chuang | +| [HDFS-9535](https://issues.apache.org/jira/browse/HDFS-9535) | Newly completed blocks in IBR should not be considered under-replicated too quickly | Major | namenode | Jing Zhao | Mingliang Liu | +| [YARN-4418](https://issues.apache.org/jira/browse/YARN-4418) | AM Resource Limit per partition can be updated to ResourceUsage as well | Major | resourcemanager | Sunil G | Sunil G | +| [YARN-4403](https://issues.apache.org/jira/browse/YARN-4403) | (AM/NM/Container)LivelinessMonitor should use monotonic time when calculating period | Critical | . | Junping Du | Junping Du | +| [YARN-4402](https://issues.apache.org/jira/browse/YARN-4402) | TestNodeManagerShutdown And TestNodeManagerResync fails with bind exception | Major | test | Brahma Reddy Battula | Brahma Reddy Battula | +| [YARN-4439](https://issues.apache.org/jira/browse/YARN-4439) | Clarify NMContainerStatus#toString method. | Major | . | Jian He | Jian He | +| [YARN-4440](https://issues.apache.org/jira/browse/YARN-4440) | FSAppAttempt#getAllowedLocalityLevelByTime should init the lastScheduler time | Major | fairscheduler | Yiqun Lin | Yiqun Lin | +| [HDFS-9516](https://issues.apache.org/jira/browse/HDFS-9516) | truncate file fails with data dirs on multiple disks | Major | datanode | Bogdan Raducanu | Plamen Jeliazkov | +| [HDFS-8894](https://issues.apache.org/jira/browse/HDFS-8894) | Set SO\_KEEPALIVE on DN server sockets | Major | datanode | Nathan Roberts | Kanaka Kumar Avvaru | +| [YARN-4461](https://issues.apache.org/jira/browse/YARN-4461) | Redundant nodeLocalityDelay log in LeafQueue | Trivial | capacityscheduler | Jason Lowe | Eric Payne | +| [YARN-4452](https://issues.apache.org/jira/browse/YARN-4452) | NPE when submit Unmanaged application | Critical | . | Naganarasimha G R | Naganarasimha G R | +| [HDFS-9565](https://issues.apache.org/jira/browse/HDFS-9565) | TestDistributedFileSystem.testLocatedFileStatusStorageIdsTypes is flaky due to race condition | Minor | fs, test | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [YARN-4225](https://issues.apache.org/jira/browse/YARN-4225) | Add preemption status to yarn queue -status for capacity scheduler | Minor | capacity scheduler, yarn | Eric Payne | Eric Payne | +| [HDFS-9515](https://issues.apache.org/jira/browse/HDFS-9515) | NPE when MiniDFSCluster#shutdown is invoked on uninitialized reference | Minor | . | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HDFS-9572](https://issues.apache.org/jira/browse/HDFS-9572) | Prevent DataNode log spam if a client connects on the data transfer port but sends no data. | Major | datanode | Chris Nauroth | Chris Nauroth | +| [HDFS-9533](https://issues.apache.org/jira/browse/HDFS-9533) | seen\_txid in the shared edits directory is modified during bootstrapping | Major | ha, namenode | Kihwal Lee | Kihwal Lee | +| [HDFS-9571](https://issues.apache.org/jira/browse/HDFS-9571) | Fix ASF Licence warnings in Jenkins reports | Major | test | Brahma Reddy Battula | Brahma Reddy Battula | +| [HDFS-9393](https://issues.apache.org/jira/browse/HDFS-9393) | After choosing favored nodes, choosing nodes for remaining replicas should go through BlockPlacementPolicy | Major | . | J.Andreina | J.Andreina | +| [HDFS-9347](https://issues.apache.org/jira/browse/HDFS-9347) | Invariant assumption in TestQuorumJournalManager.shutdown() is wrong | Major | test | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HADOOP-12656](https://issues.apache.org/jira/browse/HADOOP-12656) | MiniKdc throws "address in use" BindException | Major | . | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HADOOP-12636](https://issues.apache.org/jira/browse/HADOOP-12636) | Prevent ServiceLoader failure init for unused FileSystems | Major | fs | Inigo Goiri | Inigo Goiri | +| [MAPREDUCE-6583](https://issues.apache.org/jira/browse/MAPREDUCE-6583) | Clarify confusing sentence in MapReduce tutorial document | Minor | documentation | chris snow | Kai Sasaki | +| [HDFS-9505](https://issues.apache.org/jira/browse/HDFS-9505) | HDFS Architecture documentation needs to be refreshed. | Major | documentation | Chris Nauroth | Masatake Iwasaki | +| [YARN-4454](https://issues.apache.org/jira/browse/YARN-4454) | NM to nodelabel mapping going wrong after RM restart | Critical | . | Bibin A Chundatt | Bibin A Chundatt | +| [HDFS-9580](https://issues.apache.org/jira/browse/HDFS-9580) | TestComputeInvalidateWork#testDatanodeReRegistration failed due to unexpected number of invalidate blocks. | Major | datanode, namenode, test | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [YARN-4477](https://issues.apache.org/jira/browse/YARN-4477) | FairScheduler: Handle condition which can result in an infinite loop in attemptScheduling. | Major | fairscheduler | Tao Jie | Tao Jie | +| [HDFS-9589](https://issues.apache.org/jira/browse/HDFS-9589) | Block files which have been hardlinked should be duplicated before the DataNode appends to the them | Major | datanode | Colin P. McCabe | Colin P. McCabe | +| [HDFS-9458](https://issues.apache.org/jira/browse/HDFS-9458) | TestBackupNode always binds to port 50070, which can cause bind failures. | Major | test | Chris Nauroth | Xiao Chen | +| [YARN-4109](https://issues.apache.org/jira/browse/YARN-4109) | Exception on RM scheduler page loading with labels | Minor | . | Bibin A Chundatt | Mohammad Shahid Khan | +| [MAPREDUCE-6419](https://issues.apache.org/jira/browse/MAPREDUCE-6419) | JobHistoryServer doesn't sort properly based on Job ID when Job id's exceed 9999 | Major | webapps | Devaraj K | Mohammad Shahid Khan | +| [HDFS-9597](https://issues.apache.org/jira/browse/HDFS-9597) | BaseReplicationPolicyTest should update data node stats after adding a data node | Blocker | datanode | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HADOOP-12681](https://issues.apache.org/jira/browse/HADOOP-12681) | start-build-env.sh fails in branch-2 | Blocker | build | Akira Ajisaka | Kengo Seki | +| [HDFS-7163](https://issues.apache.org/jira/browse/HDFS-7163) | WebHdfsFileSystem should retry reads according to the configured retry policy. | Major | webhdfs | Eric Payne | Eric Payne | +| [HADOOP-12559](https://issues.apache.org/jira/browse/HADOOP-12559) | KMS connection failures should trigger TGT renewal | Major | security | Zhe Zhang | Zhe Zhang | +| [MAPREDUCE-6574](https://issues.apache.org/jira/browse/MAPREDUCE-6574) | MR AM should print host of failed tasks. | Major | . | Wangda Tan | Mohammad Shahid Khan | +| [MAPREDUCE-6589](https://issues.apache.org/jira/browse/MAPREDUCE-6589) | TestTaskLog outputs a log under directory other than target/test-dir | Major | test | Akira Ajisaka | Akira Ajisaka | +| [YARN-4315](https://issues.apache.org/jira/browse/YARN-4315) | NaN in Queue percentage for cluster apps page | Minor | . | Bibin A Chundatt | Bibin A Chundatt | +| [YARN-1382](https://issues.apache.org/jira/browse/YARN-1382) | Remove unusableRMNodesConcurrentSet (never used) in NodeListManager to get rid of memory leak | Major | resourcemanager | Alejandro Abdelnur | Rohith Sharma K S | +| [HADOOP-12682](https://issues.apache.org/jira/browse/HADOOP-12682) | Fix TestKMS#testKMSRestart\* failure | Major | . | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HADOOP-12608](https://issues.apache.org/jira/browse/HADOOP-12608) | Fix exception message in WASB when connecting with anonymous credential | Major | tools | Dushyanth | Dushyanth | +| [YARN-4524](https://issues.apache.org/jira/browse/YARN-4524) | Cleanup AppSchedulingInfo | Major | scheduler | Karthik Kambatla | Karthik Kambatla | +| [YARN-4510](https://issues.apache.org/jira/browse/YARN-4510) | Fix SLS startup failure caused by NPE | Critical | . | Bibin A Chundatt | Bibin A Chundatt | +| [MAPREDUCE-6593](https://issues.apache.org/jira/browse/MAPREDUCE-6593) | TestJobHistoryEventHandler.testTimelineEventHandling fails on trunk because of NPE | Major | . | Tsuyoshi Ozawa | Naganarasimha G R | +| [HDFS-9445](https://issues.apache.org/jira/browse/HDFS-9445) | Datanode may deadlock while handling a bad volume | Blocker | . | Kihwal Lee | Walter Su | +| [HADOOP-12658](https://issues.apache.org/jira/browse/HADOOP-12658) | Clear javadoc and check style issues around DomainSocket | Trivial | . | Kai Zheng | Kai Zheng | +| [HADOOP-12604](https://issues.apache.org/jira/browse/HADOOP-12604) | Exception may be swallowed in KMSClientProvider | Major | kms | Yongjun Zhang | Yongjun Zhang | +| [HDFS-9605](https://issues.apache.org/jira/browse/HDFS-9605) | Add links to failed volumes to explorer.html in HDFS Web UI | Minor | . | Archana T | Archana T | +| [MAPREDUCE-6577](https://issues.apache.org/jira/browse/MAPREDUCE-6577) | MR AM unable to load native library without MR\_AM\_ADMIN\_USER\_ENV set | Critical | mr-am | Sangjin Lee | Sangjin Lee | +| [HADOOP-12689](https://issues.apache.org/jira/browse/HADOOP-12689) | S3 filesystem operations stopped working correctly | Major | tools | Matthew Paduano | Matthew Paduano | +| [YARN-4546](https://issues.apache.org/jira/browse/YARN-4546) | ResourceManager crash due to scheduling opportunity overflow | Critical | resourcemanager | Jason Lowe | Jason Lowe | +| [HADOOP-12634](https://issues.apache.org/jira/browse/HADOOP-12634) | Change Lazy Rename Pending Operation Completion of WASB to address case of potential data loss due to partial copy | Critical | . | Gaurav Kanade | Gaurav Kanade | +| [HDFS-9600](https://issues.apache.org/jira/browse/HDFS-9600) | do not check replication if the block is under construction | Critical | . | Phil Yang | Phil Yang | +| [HDFS-9619](https://issues.apache.org/jira/browse/HDFS-9619) | SimulatedFSDataset sometimes can not find blockpool for the correct namenode | Major | datanode, test | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HADOOP-12675](https://issues.apache.org/jira/browse/HADOOP-12675) | Fix description about retention period in usage of expunge command | Minor | documentation | Masatake Iwasaki | Masatake Iwasaki | +| [HADOOP-12613](https://issues.apache.org/jira/browse/HADOOP-12613) | TestFind.processArguments occasionally fails | Major | test | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [MAPREDUCE-6508](https://issues.apache.org/jira/browse/MAPREDUCE-6508) | TestNetworkedJob fails consistently due to delegation token changes on RM. | Major | test | Rohith Sharma K S | Akira Ajisaka | +| [HDFS-9574](https://issues.apache.org/jira/browse/HDFS-9574) | Reduce client failures during datanode restart | Major | . | Kihwal Lee | Kihwal Lee | +| [HDFS-9493](https://issues.apache.org/jira/browse/HDFS-9493) | Test o.a.h.hdfs.server.namenode.TestMetaSave fails in trunk | Major | test | Mingliang Liu | Tony Wu | +| [HADOOP-12678](https://issues.apache.org/jira/browse/HADOOP-12678) | Handle empty rename pending metadata file during atomic rename in redo path | Critical | fs/azure | madhumita chakraborty | madhumita chakraborty | +| [HADOOP-12590](https://issues.apache.org/jira/browse/HADOOP-12590) | TestCompressorDecompressor failing without stack traces | Critical | test | Steve Loughran | John Zhuge | +| [HADOOP-12587](https://issues.apache.org/jira/browse/HADOOP-12587) | Hadoop AuthToken refuses to work without a maxinactive attribute in issued token | Blocker | security | Steve Loughran | Benoy Antony | +| [HADOOP-12551](https://issues.apache.org/jira/browse/HADOOP-12551) | Introduce FileNotFoundException for WASB FileSystem API | Major | tools | Dushyanth | Dushyanth | +| [MAPREDUCE-6068](https://issues.apache.org/jira/browse/MAPREDUCE-6068) | Illegal progress value warnings in map tasks | Major | mrv2, task | Todd Lipcon | Binglin Chang | +| [HDFS-9639](https://issues.apache.org/jira/browse/HDFS-9639) | Inconsistent Logging in BootstrapStandby | Minor | ha | BELUGA BEHR | Xiaobing Zhou | +| [HDFS-9584](https://issues.apache.org/jira/browse/HDFS-9584) | NPE in distcp when ssl configuration file does not exist in class path. | Major | distcp | Surendra Singh Lilhore | Surendra Singh Lilhore | +| [HADOOP-12584](https://issues.apache.org/jira/browse/HADOOP-12584) | Disable browsing the static directory in HttpServer2 | Major | security | Robert Kanter | Robert Kanter | +| [YARN-4567](https://issues.apache.org/jira/browse/YARN-4567) | javadoc failing on java 8 | Blocker | build | Steve Loughran | Steve Loughran | +| [YARN-4414](https://issues.apache.org/jira/browse/YARN-4414) | Nodemanager connection errors are retried at multiple levels | Major | nodemanager | Jason Lowe | Chang Li | +| [HADOOP-12603](https://issues.apache.org/jira/browse/HADOOP-12603) | TestSymlinkLocalFSFileContext#testSetTimesSymlinkToDir occasionally fail | Major | test | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [YARN-4534](https://issues.apache.org/jira/browse/YARN-4534) | Remove the redundant symbol in yarn rmadmin help msg | Trivial | . | Yiqun Lin | Yiqun Lin | +| [HADOOP-12700](https://issues.apache.org/jira/browse/HADOOP-12700) | Remove unused import in TestCompressorDecompressor.java | Minor | . | John Zhuge | John Zhuge | +| [MAPREDUCE-6601](https://issues.apache.org/jira/browse/MAPREDUCE-6601) | Fix typo in Job#setUseNewAPI | Trivial | . | Kai Sasaki | Kai Sasaki | +| [YARN-3446](https://issues.apache.org/jira/browse/YARN-3446) | FairScheduler headroom calculation should exclude nodes in the blacklist | Major | fairscheduler | zhihai xu | zhihai xu | +| [HDFS-9648](https://issues.apache.org/jira/browse/HDFS-9648) | TestStartup.testImageChecksum is broken by HDFS-9569's message change | Trivial | namenode | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HADOOP-12706](https://issues.apache.org/jira/browse/HADOOP-12706) | TestLocalFsFCStatistics#testStatisticsThreadLocalDataCleanUp times out occasionally | Major | test | Jason Lowe | Sangjin Lee | +| [YARN-4581](https://issues.apache.org/jira/browse/YARN-4581) | AHS writer thread leak makes RM crash while RM is recovering | Major | resourcemanager | sandflee | sandflee | +| [MAPREDUCE-6554](https://issues.apache.org/jira/browse/MAPREDUCE-6554) | MRAppMaster servicestart failing with NPE in MRAppMaster#parsePreviousJobHistory | Critical | . | Bibin A Chundatt | Bibin A Chundatt | +| [YARN-4389](https://issues.apache.org/jira/browse/YARN-4389) | "yarn.am.blacklisting.enabled" and "yarn.am.blacklisting.disable-failure-threshold" should be app specific rather than a setting for whole YARN cluster | Critical | applications | Junping Du | Sunil G | +| [HDFS-9612](https://issues.apache.org/jira/browse/HDFS-9612) | DistCp worker threads are not terminated after jobs are done. | Major | distcp | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HADOOP-12712](https://issues.apache.org/jira/browse/HADOOP-12712) | Fix some cmake plugin and native build warnings | Minor | native | Colin P. McCabe | Colin P. McCabe | +| [YARN-4538](https://issues.apache.org/jira/browse/YARN-4538) | QueueMetrics pending cores and memory metrics wrong | Major | . | Bibin A Chundatt | Bibin A Chundatt | +| [YARN-4596](https://issues.apache.org/jira/browse/YARN-4596) | SystemMetricPublisher should not swallow error messages from TimelineClient#putEntities | Major | timelineserver | Li Lu | Li Lu | +| [YARN-4502](https://issues.apache.org/jira/browse/YARN-4502) | Fix two AM containers get allocated when AM restart | Critical | . | Yesha Vora | Vinod Kumar Vavilapalli | +| [HDFS-9623](https://issues.apache.org/jira/browse/HDFS-9623) | Update example configuration of block state change log in log4j.properties | Minor | logging | Masatake Iwasaki | Masatake Iwasaki | +| [YARN-4565](https://issues.apache.org/jira/browse/YARN-4565) | When sizeBasedWeight enabled for FairOrderingPolicy in CapacityScheduler, Sometimes lead to situation where all queue resources consumed by AMs only | Major | capacity scheduler, capacityscheduler | Karam Singh | Wangda Tan | +| [HADOOP-12356](https://issues.apache.org/jira/browse/HADOOP-12356) | Fix computing CPU usage statistics on Windows | Major | util | Yunqi Zhang | Inigo Goiri | +| [HDFS-9661](https://issues.apache.org/jira/browse/HDFS-9661) | Deadlock in DN.FsDatasetImpl between moveBlockAcrossStorage and createRbw | Major | datanode | ade | ade | +| [HDFS-9655](https://issues.apache.org/jira/browse/HDFS-9655) | NN should start JVM pause monitor before loading fsimage | Critical | . | John Zhuge | John Zhuge | +| [YARN-4559](https://issues.apache.org/jira/browse/YARN-4559) | Make leader elector and zk store share the same curator client | Major | . | Jian He | Jian He | +| [HADOOP-12605](https://issues.apache.org/jira/browse/HADOOP-12605) | Fix intermittent failure of TestIPC.testIpcWithReaderQueuing | Minor | test | Masatake Iwasaki | Masatake Iwasaki | +| [HDFS-9625](https://issues.apache.org/jira/browse/HDFS-9625) | set replication for empty file failed when set storage policy | Major | namenode | DENG FEI | DENG FEI | +| [HADOOP-12423](https://issues.apache.org/jira/browse/HADOOP-12423) | Handle failure of registering shutdownhook by ShutdownHookManager in static block | Minor | fs | Abhishek Agarwal | Abhishek Agarwal | +| [HDFS-9634](https://issues.apache.org/jira/browse/HDFS-9634) | webhdfs client side exceptions don't provide enough details | Major | webhdfs | Eric Payne | Eric Payne | +| [YARN-4608](https://issues.apache.org/jira/browse/YARN-4608) | Redundant code statement in WritingYarnApplications | Minor | documentation | Kai Sasaki | Kai Sasaki | +| [HADOOP-7161](https://issues.apache.org/jira/browse/HADOOP-7161) | Remove unnecessary oro package from dependency management section | Minor | build | Todd Lipcon | Sean Busbey | +| [YARN-4610](https://issues.apache.org/jira/browse/YARN-4610) | Reservations continue looking for one app causes other apps to starve | Blocker | capacityscheduler | Jason Lowe | Jason Lowe | +| [HADOOP-12659](https://issues.apache.org/jira/browse/HADOOP-12659) | Incorrect usage of config parameters in token manager of KMS | Major | security | Tianyin Xu | Mingliang Liu | +| [MAPREDUCE-6605](https://issues.apache.org/jira/browse/MAPREDUCE-6605) | Fix typos mapreduce.map.skip.proc.count.autoincr and mapreduce.reduce.skip.proc.count.autoincr in mapred-default.xml | Major | documentation | Dong Zhen | Kai Sasaki | +| [YARN-4605](https://issues.apache.org/jira/browse/YARN-4605) | Spelling mistake in the help message of "yarn applicationattempt" command | Trivial | client, yarn | Manjunath Ballur | Weiwei Yang | +| [HDFS-9682](https://issues.apache.org/jira/browse/HDFS-9682) | Fix a typo "aplication" in HttpFS document | Trivial | documentation | Weiwei Yang | Weiwei Yang | +| [HADOOP-12730](https://issues.apache.org/jira/browse/HADOOP-12730) | Hadoop streaming -mapper and -reducer options are wrongly documented as required | Major | documentation | DeepakVohra | Kengo Seki | +| [HDFS-8898](https://issues.apache.org/jira/browse/HDFS-8898) | Create API and command-line argument to get quota and quota usage without detailed content summary | Major | fs | Joep Rottinghuis | Ming Ma | +| [YARN-4598](https://issues.apache.org/jira/browse/YARN-4598) | Invalid event: RESOURCE\_FAILED at CONTAINER\_CLEANEDUP\_AFTER\_KILL | Major | nodemanager | tangshangwen | tangshangwen | +| [YARN-4592](https://issues.apache.org/jira/browse/YARN-4592) | Remove unused GetContainerStatus proto | Minor | . | Chang Li | Chang Li | +| [YARN-4520](https://issues.apache.org/jira/browse/YARN-4520) | FinishAppEvent is leaked in leveldb if no app's container running on this node | Major | nodemanager | sandflee | sandflee | +| [MAPREDUCE-6610](https://issues.apache.org/jira/browse/MAPREDUCE-6610) | JobHistoryEventHandler should not swallow timeline response | Trivial | . | Li Lu | Li Lu | +| [HDFS-9690](https://issues.apache.org/jira/browse/HDFS-9690) | ClientProtocol.addBlock is not idempotent after HDFS-8071 | Major | namenode | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze | +| [HADOOP-12743](https://issues.apache.org/jira/browse/HADOOP-12743) | Fix git environment check during test-patch | Major | . | Ray Chiang | Allen Wittenauer | +| [HADOOP-12735](https://issues.apache.org/jira/browse/HADOOP-12735) | core-default.xml misspells hadoop.workaround.non.threadsafe.getpwuid | Minor | . | Ray Chiang | Ray Chiang | +| [MAPREDUCE-6619](https://issues.apache.org/jira/browse/MAPREDUCE-6619) | HADOOP\_CLASSPATH is overwritten in MR container | Major | mrv2 | shanyu zhao | Junping Du | +| [HDFS-8999](https://issues.apache.org/jira/browse/HDFS-8999) | Allow a file to be closed with COMMITTED but not yet COMPLETE blocks. | Major | namenode | Jitendra Nath Pandey | Tsz Wo Nicholas Sze | +| [MAPREDUCE-6595](https://issues.apache.org/jira/browse/MAPREDUCE-6595) | Fix findbugs warnings in OutputCommitter and FileOutputCommitter | Major | . | Akira Ajisaka | Akira Ajisaka | +| [MAPREDUCE-6563](https://issues.apache.org/jira/browse/MAPREDUCE-6563) | Streaming documentation contains a stray '%' character. | Trivial | documentation | Chris Nauroth | Chris Nauroth | +| [YARN-4519](https://issues.apache.org/jira/browse/YARN-4519) | potential deadlock of CapacityScheduler between decrease container and assign containers | Major | capacityscheduler | sandflee | MENG DING | +| [MAPREDUCE-6616](https://issues.apache.org/jira/browse/MAPREDUCE-6616) | Fail to create jobhistory file if there are some multibyte characters in the job name | Major | jobhistoryserver | Akira Ajisaka | Kousuke Saruta | +| [YARN-4411](https://issues.apache.org/jira/browse/YARN-4411) | RMAppAttemptImpl#createApplicationAttemptReport throws IllegalArgumentException | Major | resourcemanager | yarntime | Bibin A Chundatt | +| [YARN-4617](https://issues.apache.org/jira/browse/YARN-4617) | LeafQueue#pendingOrderingPolicy should always use fixed ordering policy instead of using same as active applications ordering policy | Major | capacity scheduler | Rohith Sharma K S | Rohith Sharma K S | +| [YARN-4428](https://issues.apache.org/jira/browse/YARN-4428) | Redirect RM page to AHS page when AHS turned on and RM page is not available | Major | . | Chang Li | Chang Li | +| [MAPREDUCE-6618](https://issues.apache.org/jira/browse/MAPREDUCE-6618) | YarnClientProtocolProvider leaking the YarnClient thread. | Major | . | Xuan Gong | Xuan Gong | +| [HDFS-9210](https://issues.apache.org/jira/browse/HDFS-9210) | Fix some misuse of %n in VolumeScanner#printStats | Minor | datanode | Xiaoyu Yao | Xiaoyu Yao | +| [HDFS-9701](https://issues.apache.org/jira/browse/HDFS-9701) | DN may deadlock when hot-swapping under load | Major | . | Xiao Chen | Xiao Chen | +| [YARN-3102](https://issues.apache.org/jira/browse/YARN-3102) | Decommisioned Nodes not listed in Web UI | Minor | resourcemanager | Bibin A Chundatt | Kuhu Shukla | +| [HDFS-9406](https://issues.apache.org/jira/browse/HDFS-9406) | FSImage may get corrupted after deleting snapshot | Major | namenode | Stanislav Antic | Yongjun Zhang | +| [HDFS-9718](https://issues.apache.org/jira/browse/HDFS-9718) | HAUtil#getConfForOtherNodes should unset independent generic keys before initialize | Major | namenode | DENG FEI | DENG FEI | +| [HDFS-9708](https://issues.apache.org/jira/browse/HDFS-9708) | FSNamesystem.initAuditLoggers() doesn't trim classnames | Minor | fs | Steve Loughran | Mingliang Liu | +| [MAPREDUCE-6621](https://issues.apache.org/jira/browse/MAPREDUCE-6621) | Memory Leak in JobClient#submitJobInternal() | Major | . | Xuan Gong | Xuan Gong | +| [HADOOP-12755](https://issues.apache.org/jira/browse/HADOOP-12755) | Fix typo in defaultFS warning message | Trivial | . | Andrew Wang | Andrew Wang | +| [HDFS-9739](https://issues.apache.org/jira/browse/HDFS-9739) | DatanodeStorage.isValidStorageId() is broken | Critical | hdfs-client | Kihwal Lee | Mingliang Liu | +| [HDFS-9740](https://issues.apache.org/jira/browse/HDFS-9740) | Use a reasonable limit in DFSTestUtil.waitForMetric() | Major | test | Kihwal Lee | Chang Li | +| [HADOOP-12761](https://issues.apache.org/jira/browse/HADOOP-12761) | incremental maven build is not really incremental | Minor | build | Sangjin Lee | Sangjin Lee | +| [HDFS-9748](https://issues.apache.org/jira/browse/HDFS-9748) | When addExpectedReplicasToPending is called twice, pendingReplications should avoid duplication | Minor | . | Walter Su | Walter Su | +| [HDFS-9730](https://issues.apache.org/jira/browse/HDFS-9730) | Storage ID update does not happen when there is a layout change | Major | datanode | Kihwal Lee | Tsz Wo Nicholas Sze | +| [HDFS-9724](https://issues.apache.org/jira/browse/HDFS-9724) | Degraded performance in WebHDFS listing as it does not reuse ObjectMapper | Critical | performance | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-12766](https://issues.apache.org/jira/browse/HADOOP-12766) | The default value of "hadoop.workaround.non.threadsafe.getpwuid" is different between core-default.xml and NativeIO.java | Minor | . | Akira Ajisaka | Akira Ajisaka | +| [HDFS-9761](https://issues.apache.org/jira/browse/HDFS-9761) | Rebalancer sleeps too long between iterations | Blocker | balancer & mover | Adrian Bridgett | Mingliang Liu | +| [HDFS-9713](https://issues.apache.org/jira/browse/HDFS-9713) | DataXceiver#copyBlock should return if block is pinned | Major | datanode | Uma Maheswara Rao G | Uma Maheswara Rao G | +| [HADOOP-12773](https://issues.apache.org/jira/browse/HADOOP-12773) | HBase classes fail to load with client/job classloader enabled | Major | util | Sangjin Lee | Sangjin Lee | +| [HDFS-9777](https://issues.apache.org/jira/browse/HDFS-9777) | Fix typos in DFSAdmin command line and documentation | Trivial | hdfs-client | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HDFS-9784](https://issues.apache.org/jira/browse/HDFS-9784) | Example usage is not correct in Transparent Encryption document | Major | documentation | Takashi Ohnishi | Takashi Ohnishi | +| [HDFS-9752](https://issues.apache.org/jira/browse/HDFS-9752) | Permanent write failures may happen to slow writers during datanode rolling upgrades | Critical | . | Kihwal Lee | Walter Su | +| [HDFS-9760](https://issues.apache.org/jira/browse/HDFS-9760) | WebHDFS AuthFilter cannot be configured with custom AltKerberos auth handler | Major | webhdfs | Ryan Sasson | Ryan Sasson | +| [HDFS-9779](https://issues.apache.org/jira/browse/HDFS-9779) | TestReplicationPolicyWithNodeGroup NODE variable picks wrong rack value | Minor | test | Kuhu Shukla | Kuhu Shukla | +| [HADOOP-12792](https://issues.apache.org/jira/browse/HADOOP-12792) | TestUserGroupInformation#testGetServerSideGroups fails in chroot | Minor | security, test | Eric Badger | Eric Badger | +| [HDFS-9788](https://issues.apache.org/jira/browse/HDFS-9788) | Incompatible tag renumbering in HeartbeatResponseProto | Blocker | rolling upgrades | Andrew Wang | Andrew Wang | +| [HADOOP-12795](https://issues.apache.org/jira/browse/HADOOP-12795) | KMS does not log detailed stack trace for unexpected errors. | Major | kms | Chris Nauroth | Chris Nauroth | +| [HADOOP-12699](https://issues.apache.org/jira/browse/HADOOP-12699) | TestKMS#testKMSProvider intermittently fails during 'test rollover draining' | Major | . | Xiao Chen | Xiao Chen | +| [HDFS-9790](https://issues.apache.org/jira/browse/HDFS-9790) | HDFS Balancer should exit with a proper message if upgrade is not finalized | Major | . | Xiaobing Zhou | Xiaobing Zhou | +| [HDFS-9801](https://issues.apache.org/jira/browse/HDFS-9801) | ReconfigurableBase should update the cached configuration | Major | datanode | Arpit Agarwal | Arpit Agarwal | +| [HADOOP-12780](https://issues.apache.org/jira/browse/HADOOP-12780) | During WASB atomic rename handle crash when one directory has been renamed but not file under it. | Critical | fs/azure | madhumita chakraborty | madhumita chakraborty | +| [HADOOP-12589](https://issues.apache.org/jira/browse/HADOOP-12589) | Fix intermittent test failure of TestCopyPreserveFlag | Major | test | Tsuyoshi Ozawa | Masatake Iwasaki | +| [HADOOP-12786](https://issues.apache.org/jira/browse/HADOOP-12786) | "hadoop key" command usage is not documented | Major | documentation | Akira Ajisaka | Xiao Chen | +| [HDFS-9765](https://issues.apache.org/jira/browse/HDFS-9765) | TestBlockScanner#testVolumeIteratorWithCaching fails intermittently | Major | test | Mingliang Liu | Akira Ajisaka | +| [HDFS-9456](https://issues.apache.org/jira/browse/HDFS-9456) | BlockPlacementPolicyWithNodeGroup should override verifyBlockPlacement | Major | . | Junping Du | Xiaobing Zhou | +| [HADOOP-12810](https://issues.apache.org/jira/browse/HADOOP-12810) | FileSystem#listLocatedStatus causes unnecessary RPC calls | Major | fs, fs/s3 | Ryan Blue | Ryan Blue | +| [MAPREDUCE-6341](https://issues.apache.org/jira/browse/MAPREDUCE-6341) | Fix typo in mapreduce tutorial | Trivial | . | John Michael Luy | John Michael Luy | +| [HADOOP-12787](https://issues.apache.org/jira/browse/HADOOP-12787) | KMS SPNEGO sequence does not work with WEBHDFS | Major | kms, security | Xiaoyu Yao | Xiaoyu Yao | +| [HDFS-9815](https://issues.apache.org/jira/browse/HDFS-9815) | Move o.a.h.fs.Hdfs to hadoop-hdfs-client | Blocker | . | Haohui Mai | Vinayakumar B | +| [HDFS-9799](https://issues.apache.org/jira/browse/HDFS-9799) | Reimplement getCurrentTrashDir to remove incompatibility | Blocker | . | Zhe Zhang | Zhe Zhang | +| [YARN-4654](https://issues.apache.org/jira/browse/YARN-4654) | Yarn node label CLI should parse "=" correctly when trying to remove all labels on a node | Major | . | Wangda Tan | Naganarasimha G R | +| [HDFS-6832](https://issues.apache.org/jira/browse/HDFS-6832) | Fix the usage of 'hdfs namenode' command | Minor | . | Akira Ajisaka | Manjunath Ballur | +| [HDFS-8923](https://issues.apache.org/jira/browse/HDFS-8923) | Add -source flag to balancer usage message | Trivial | balancer & mover, documentation | Chris Trezzo | Chris Trezzo | +| [HDFS-9764](https://issues.apache.org/jira/browse/HDFS-9764) | DistCp doesn't print value for several arguments including -numListstatusThreads | Minor | distcp | Yongjun Zhang | Wei-Chiu Chuang | +| [MAPREDUCE-6637](https://issues.apache.org/jira/browse/MAPREDUCE-6637) | Testcase Failure : TestFileInputFormat.testSplitLocationInfo | Major | test | Brahma Reddy Battula | Brahma Reddy Battula | +| [HDFS-9839](https://issues.apache.org/jira/browse/HDFS-9839) | Reduce verbosity of processReport logging | Major | namenode | Arpit Agarwal | Arpit Agarwal | +| [HDFS-7452](https://issues.apache.org/jira/browse/HDFS-7452) | skip StandbyException log for getCorruptFiles() | Minor | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [YARN-4707](https://issues.apache.org/jira/browse/YARN-4707) | Remove the extra char (\>) from SecureContainer.md | Major | documentation | Brahma Reddy Battula | Brahma Reddy Battula | +| [YARN-4386](https://issues.apache.org/jira/browse/YARN-4386) | refreshNodesGracefully() should send recommission event to active RMNodes only | Minor | graceful | Kuhu Shukla | Kuhu Shukla | +| [HDFS-9842](https://issues.apache.org/jira/browse/HDFS-9842) | dfs.datanode.balance.bandwidthPerSec should accept friendly size units | Minor | balancer & mover | Yiqun Lin | Yiqun Lin | +| [YARN-4709](https://issues.apache.org/jira/browse/YARN-4709) | NMWebServices produces incorrect JSON for containers | Critical | . | Varun Saxena | Varun Saxena | +| [MAPREDUCE-6635](https://issues.apache.org/jira/browse/MAPREDUCE-6635) | Unsafe long to int conversion in UncompressedSplitLineReader and IndexOutOfBoundsException | Critical | . | Sergey Shelukhin | Junping Du | +| [HDFS-9549](https://issues.apache.org/jira/browse/HDFS-9549) | TestCacheDirectives#testExceedsCapacity is flaky | Major | . | Wei-Chiu Chuang | Xiao Chen | +| [YARN-2046](https://issues.apache.org/jira/browse/YARN-2046) | Out of band heartbeats are sent only on container kill and possibly too early | Major | nodemanager | Jason Lowe | Ming Ma | +| [HDFS-9844](https://issues.apache.org/jira/browse/HDFS-9844) | Correct path creation in getTrashRoot to handle root dir | Blocker | encryption | Zhe Zhang | Zhe Zhang | +| [YARN-4722](https://issues.apache.org/jira/browse/YARN-4722) | AsyncDispatcher logs redundant event queue sizes | Major | . | Jason Lowe | Jason Lowe | +| [HADOOP-12716](https://issues.apache.org/jira/browse/HADOOP-12716) | KerberosAuthenticator#doSpnegoSequence use incorrect class to determine isKeyTab in JDK8 | Major | security | Xiaoyu Yao | Xiaoyu Yao | +| [HDFS-9855](https://issues.apache.org/jira/browse/HDFS-9855) | Modify TestAuditLoggerWithCommands to workaround the absence of HDFS-8332 | Major | test | Kuhu Shukla | Kuhu Shukla | +| [YARN-4723](https://issues.apache.org/jira/browse/YARN-4723) | NodesListManager$UnknownNodeId ClassCastException | Critical | resourcemanager | Jason Lowe | Kuhu Shukla | +| [HADOOP-12849](https://issues.apache.org/jira/browse/HADOOP-12849) | TestSymlinkLocalFSFileSystem fails intermittently | Major | test | Mingliang Liu | Mingliang Liu | +| [HADOOP-12831](https://issues.apache.org/jira/browse/HADOOP-12831) | LocalFS/FSOutputSummer NPEs in constructor if bytes per checksum set to 0 | Minor | fs | Steve Loughran | Mingliang Liu | +| [HADOOP-12846](https://issues.apache.org/jira/browse/HADOOP-12846) | Credential Provider Recursive Dependencies | Major | . | Larry McCay | Larry McCay | +| [HDFS-9864](https://issues.apache.org/jira/browse/HDFS-9864) | Correct reference for RENEWDELEGATIONTOKEN and CANCELDELEGATIONTOKEN in webhdfs doc | Major | documentation | Brahma Reddy Battula | Brahma Reddy Battula | +| [HADOOP-12622](https://issues.apache.org/jira/browse/HADOOP-12622) | RetryPolicies (other than FailoverOnNetworkExceptionRetry) should put on retry failed reason or the log from RMProxy's retry could be very misleading. | Critical | auto-failover | Junping Du | Junping Du | +| [YARN-4748](https://issues.apache.org/jira/browse/YARN-4748) | ApplicationHistoryManagerOnTimelineStore should not swallow exceptions on generateApplicationReport | Major | timelineserver | Li Lu | Li Lu | +| [HADOOP-12851](https://issues.apache.org/jira/browse/HADOOP-12851) | S3AFileSystem Uptake of ProviderUtils.excludeIncompatibleCredentialProviders | Major | fs/s3 | Larry McCay | Larry McCay | +| [HADOOP-12843](https://issues.apache.org/jira/browse/HADOOP-12843) | Fix findbugs warnings in hadoop-common (branch-2) | Major | . | Akira Ajisaka | Akira Ajisaka | +| [HDFS-9870](https://issues.apache.org/jira/browse/HDFS-9870) | Remove unused imports from DFSUtil | Major | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [HDFS-8791](https://issues.apache.org/jira/browse/HDFS-8791) | block ID-based DN storage layout can be very slow for datanode on ext4 | Blocker | datanode | Nathan Roberts | Chris Trezzo | +| [HDFS-9880](https://issues.apache.org/jira/browse/HDFS-9880) | TestDatanodeRegistration fails occasionally | Major | test | Kihwal Lee | Kihwal Lee | +| [HDFS-9881](https://issues.apache.org/jira/browse/HDFS-9881) | DistributedFileSystem#getTrashRoot returns incorrect path for encryption zones | Critical | . | Andrew Wang | Andrew Wang | +| [HDFS-9766](https://issues.apache.org/jira/browse/HDFS-9766) | TestDataNodeMetrics#testDataNodeTimeSpend fails intermittently | Major | test | Mingliang Liu | Xiao Chen | +| [HDFS-9851](https://issues.apache.org/jira/browse/HDFS-9851) | Name node throws NPE when setPermission is called on a path that does not exist | Critical | namenode | David Yan | Brahma Reddy Battula | +| [HDFS-9886](https://issues.apache.org/jira/browse/HDFS-9886) | Configuration properties for hedged read is broken | Blocker | . | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-12870](https://issues.apache.org/jira/browse/HADOOP-12870) | Fix typo admininistration in CommandsManual.md | Minor | documentation | Akira Ajisaka | John Zhuge | +| [HDFS-9048](https://issues.apache.org/jira/browse/HDFS-9048) | DistCp documentation is out-of-dated | Major | . | Haohui Mai | Daisuke Kobayashi | +| [HADOOP-12871](https://issues.apache.org/jira/browse/HADOOP-12871) | Fix dead link to NativeLibraries.html in CommandsManual.md | Minor | documentation | Akira Ajisaka | Brahma Reddy Battula | +| [HADOOP-12872](https://issues.apache.org/jira/browse/HADOOP-12872) | Fix formatting in ServiceLevelAuth.md | Trivial | documentation | Akira Ajisaka | Brahma Reddy Battula | +| [MAPREDUCE-4785](https://issues.apache.org/jira/browse/MAPREDUCE-4785) | TestMRApp occasionally fails | Major | mrv2, test | Jason Lowe | Haibo Chen | +| [HADOOP-12717](https://issues.apache.org/jira/browse/HADOOP-12717) | NPE when trying to rename a directory in Windows Azure Storage FileSystem | Blocker | . | Robert Yokota | Robert Yokota | +| [YARN-4763](https://issues.apache.org/jira/browse/YARN-4763) | RMApps Page crashes with NPE | Major | webapp | Bibin A Chundatt | Bibin A Chundatt | +| [YARN-4761](https://issues.apache.org/jira/browse/YARN-4761) | NMs reconnecting with changed capabilities can lead to wrong cluster resource calculations on fair scheduler | Major | fairscheduler | Sangjin Lee | Sangjin Lee | +| [YARN-4744](https://issues.apache.org/jira/browse/YARN-4744) | Too many signal to container failure in case of LCE | Major | . | Bibin A Chundatt | Sidharta Seethana | +| [YARN-4760](https://issues.apache.org/jira/browse/YARN-4760) | proxy redirect to history server uses wrong URL | Major | webapp | Jason Lowe | Eric Badger | +| [HDFS-9865](https://issues.apache.org/jira/browse/HDFS-9865) | TestBlockReplacement fails intermittently in trunk | Major | test | Yiqun Lin | Yiqun Lin | +| [HDFS-9812](https://issues.apache.org/jira/browse/HDFS-9812) | Streamer threads leak if failure happens when closing DFSOutputStream | Major | hdfs-client | Yiqun Lin | Yiqun Lin | +| [HADOOP-12688](https://issues.apache.org/jira/browse/HADOOP-12688) | Fix deadlinks in Compatibility.md | Major | documentation | Akira Ajisaka | Gabor Liptak | +| [HADOOP-12903](https://issues.apache.org/jira/browse/HADOOP-12903) | IPC Server should allow suppressing exception logging by type, not log 'server too busy' messages | Major | ipc | Arpit Agarwal | Arpit Agarwal | +| [HDFS-9934](https://issues.apache.org/jira/browse/HDFS-9934) | ReverseXML oiv processor should bail out if the XML file's layoutVersion doesn't match oiv's | Major | tools | Colin P. McCabe | Colin P. McCabe | +| [HDFS-9933](https://issues.apache.org/jira/browse/HDFS-9933) | ReverseXML should be capitalized in oiv usage message | Minor | tools | Colin P. McCabe | Colin P. McCabe | +| [HDFS-9953](https://issues.apache.org/jira/browse/HDFS-9953) | Download File from UI broken after pagination | Blocker | namenode | Brahma Reddy Battula | Brahma Reddy Battula | +| [HDFS-9904](https://issues.apache.org/jira/browse/HDFS-9904) | testCheckpointCancellationDuringUpload occasionally fails | Major | test | Kihwal Lee | Yiqun Lin | +| [MAPREDUCE-6579](https://issues.apache.org/jira/browse/MAPREDUCE-6579) | JobStatus#getFailureInfo should not output diagnostic information when the job is running | Blocker | test | Rohith Sharma K S | Akira Ajisaka | +| [MAPREDUCE-6645](https://issues.apache.org/jira/browse/MAPREDUCE-6645) | TestWordStats outputs logs under directories other than target/test-dir | Major | test | Akira Ajisaka | Gabor Liptak | +| [HDFS-9874](https://issues.apache.org/jira/browse/HDFS-9874) | Long living DataXceiver threads cause volume shutdown to block. | Critical | datanode | Rushabh S Shah | Rushabh S Shah | +| [HDFS-3677](https://issues.apache.org/jira/browse/HDFS-3677) | dfs.namenode.edits.dir.required missing from hdfs-default.xml | Major | documentation, namenode | Todd Lipcon | Mark Yang | +| [HDFS-7166](https://issues.apache.org/jira/browse/HDFS-7166) | SbNN Web UI shows #Under replicated blocks and #pending deletion blocks | Major | ha | Juan Yu | Wei-Chiu Chuang | +| [YARN-4686](https://issues.apache.org/jira/browse/YARN-4686) | MiniYARNCluster.start() returns before cluster is completely started | Major | test | Rohith Sharma K S | Eric Badger | +| [MAPREDUCE-6363](https://issues.apache.org/jira/browse/MAPREDUCE-6363) | [NNBench] Lease mismatch error when running with multiple mappers | Critical | benchmarks | Brahma Reddy Battula | Bibin A Chundatt | +| [HDFS-10189](https://issues.apache.org/jira/browse/HDFS-10189) | PacketResponder#toString should include the downstreams for PacketResponderType.HAS\_DOWNSTREAM\_IN\_PIPELINE | Minor | datanode | Joe Pallas | Joe Pallas | +| [MAPREDUCE-6580](https://issues.apache.org/jira/browse/MAPREDUCE-6580) | Test failure : TestMRJobsWithProfiler | Major | . | Rohith Sharma K S | Eric Badger | +| [MAPREDUCE-6656](https://issues.apache.org/jira/browse/MAPREDUCE-6656) | [NNBench] OP\_DELETE operation isn't working after MAPREDUCE-6363 | Blocker | . | J.Andreina | J.Andreina | +| [HDFS-10193](https://issues.apache.org/jira/browse/HDFS-10193) | fuse\_dfs segfaults if uid cannot be resolved to a username | Major | fuse-dfs | John Thiltges | John Thiltges | +| [HDFS-10199](https://issues.apache.org/jira/browse/HDFS-10199) | Unit tests TestCopyFiles, TestDistCh, TestLogalyzer under org.apache.hadoop.tools are failing | Minor | . | Tibor Kiss | Tibor Kiss | +| [YARN-4820](https://issues.apache.org/jira/browse/YARN-4820) | ResourceManager web redirects in HA mode drops query parameters | Major | . | Varun Vasudev | Varun Vasudev | +| [YARN-4850](https://issues.apache.org/jira/browse/YARN-4850) | test-fair-scheduler.xml isn't valid xml | Blocker | fairscheduler, test | Allen Wittenauer | Yufei Gu | +| [HADOOP-12962](https://issues.apache.org/jira/browse/HADOOP-12962) | KMS key names are incorrectly encoded when creating key | Major | . | Xiao Chen | Xiao Chen | +| [HADOOP-12958](https://issues.apache.org/jira/browse/HADOOP-12958) | PhantomReference for filesystem statistics can trigger OOM | Major | . | Jason Lowe | Sangjin Lee | +| [HADOOP-12873](https://issues.apache.org/jira/browse/HADOOP-12873) | Remove MRv1 terms from HttpAuthentication.md | Major | documentation | Akira Ajisaka | Brahma Reddy Battula | +| [HDFS-10182](https://issues.apache.org/jira/browse/HDFS-10182) | Hedged read might overwrite user's buf | Major | . | zhouyingchao | zhouyingchao | +| [YARN-4773](https://issues.apache.org/jira/browse/YARN-4773) | Log aggregation performs extraneous filesystem operations when rolling log aggregation is disabled | Minor | nodemanager | Jason Lowe | Jun Gong | +| [MAPREDUCE-6662](https://issues.apache.org/jira/browse/MAPREDUCE-6662) | Clear ASF Warnings on test data files | Minor | . | Vinayakumar B | Vinayakumar B | +| [HDFS-9871](https://issues.apache.org/jira/browse/HDFS-9871) | "Bytes Being Moved" -ve(-1 B) when cluster was already balanced. | Major | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [YARN-4863](https://issues.apache.org/jira/browse/YARN-4863) | AHS Security login should be in serviceInit() instead of serviceStart() | Major | timelineserver | Junping Du | Junping Du | +| [HDFS-10197](https://issues.apache.org/jira/browse/HDFS-10197) | TestFsDatasetCache failing intermittently due to timeout | Major | test | Yiqun Lin | Yiqun Lin | +| [HDFS-9478](https://issues.apache.org/jira/browse/HDFS-9478) | Reason for failing ipc.FairCallQueue contruction should be thrown | Minor | . | Archana T | Ajith S | +| [HDFS-10228](https://issues.apache.org/jira/browse/HDFS-10228) | TestHDFSCLI fails | Major | test | Akira Ajisaka | Akira Ajisaka | +| [YARN-4865](https://issues.apache.org/jira/browse/YARN-4865) | Track Reserved resources in ResourceUsage and QueueCapacities | Major | resourcemanager | Sunil G | Sunil G | +| [HADOOP-12972](https://issues.apache.org/jira/browse/HADOOP-12972) | Lz4Compressor#getLibraryName returns the wrong version number | Trivial | native | John Zhuge | Colin P. McCabe | +| [HDFS-5177](https://issues.apache.org/jira/browse/HDFS-5177) | blocksScheduled count should be decremented for abandoned blocks | Major | namenode | Vinayakumar B | Vinayakumar B | +| [HDFS-10223](https://issues.apache.org/jira/browse/HDFS-10223) | peerFromSocketAndKey performs SASL exchange before setting connection timeouts | Major | . | Colin P. McCabe | Colin P. McCabe | +| [HDFS-10221](https://issues.apache.org/jira/browse/HDFS-10221) | Add .json to the rat exclusions | Blocker | build | Ming Ma | Ming Ma | +| [HADOOP-12902](https://issues.apache.org/jira/browse/HADOOP-12902) | JavaDocs for SignerSecretProvider are out-of-date in AuthenticationFilter | Major | documentation | Robert Kanter | Gabor Liptak | +| [YARN-4183](https://issues.apache.org/jira/browse/YARN-4183) | Clarify the behavior of timeline service config properties | Major | . | Mit Desai | Naganarasimha G R | +| [HDFS-10253](https://issues.apache.org/jira/browse/HDFS-10253) | Fix TestRefreshCallQueue failure. | Major | . | Brahma Reddy Battula | Xiaoyu Yao | +| [YARN-4746](https://issues.apache.org/jira/browse/YARN-4746) | yarn web services should convert parse failures of appId, appAttemptId and containerId to 400 | Minor | webapp | Steve Loughran | Bibin A Chundatt | +| [HDFS-9599](https://issues.apache.org/jira/browse/HDFS-9599) | TestDecommissioningStatus.testDecommissionStatus occasionally fails | Major | namenode | Wei-Chiu Chuang | Yiqun Lin | +| [YARN-4706](https://issues.apache.org/jira/browse/YARN-4706) | UI Hosting Configuration in TimelineServer doc is broken | Critical | documentation | Akira Ajisaka | Akira Ajisaka | +| [HDFS-10178](https://issues.apache.org/jira/browse/HDFS-10178) | Permanent write failures can happen if pipeline recoveries occur for the first packet | Critical | . | Kihwal Lee | Kihwal Lee | +| [HDFS-8496](https://issues.apache.org/jira/browse/HDFS-8496) | Calling stopWriter() with FSDatasetImpl lock held may block other threads | Major | . | zhouyingchao | Colin P. McCabe | +| [HDFS-9917](https://issues.apache.org/jira/browse/HDFS-9917) | IBR accumulate more objects when SNN was down for sometime. | Critical | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [HDFS-10239](https://issues.apache.org/jira/browse/HDFS-10239) | Fsshell mv fails if port usage doesn't match in src and destination paths | Major | . | Kuhu Shukla | Kuhu Shukla | +| [YARN-4893](https://issues.apache.org/jira/browse/YARN-4893) | Fix some intermittent test failures in TestRMAdminService | Blocker | . | Junping Du | Brahma Reddy Battula | +| [YARN-4916](https://issues.apache.org/jira/browse/YARN-4916) | TestNMProxy.tesNMProxyRPCRetry fails. | Minor | . | Tibor Kiss | Tibor Kiss | +| [YARN-4915](https://issues.apache.org/jira/browse/YARN-4915) | Fix typo in YARN Secure Containers documentation | Trivial | documentation, yarn | Takashi Ohnishi | Takashi Ohnishi | +| [YARN-4917](https://issues.apache.org/jira/browse/YARN-4917) | Fix typos in documentation of Capacity Scheduler. | Minor | documentation | Takashi Ohnishi | Takashi Ohnishi | +| [HDFS-10261](https://issues.apache.org/jira/browse/HDFS-10261) | TestBookKeeperHACheckpoints doesn't handle ephemeral HTTP ports | Major | . | Eric Badger | Eric Badger | +| [YARN-4699](https://issues.apache.org/jira/browse/YARN-4699) | Scheduler UI and REST o/p is not in sync when -replaceLabelsOnNode is used to change label of a node | Critical | capacity scheduler | Sunil G | Sunil G | +| [HADOOP-12022](https://issues.apache.org/jira/browse/HADOOP-12022) | fix site -Pdocs -Pdist in hadoop-project-dist; cleanout remaining forrest bits | Blocker | build | Allen Wittenauer | Allen Wittenauer | +| [MAPREDUCE-6670](https://issues.apache.org/jira/browse/MAPREDUCE-6670) | TestJobListCache#testEviction sometimes fails on Windows with timeout | Minor | test | Gergely Novák | Gergely Novák | +| [HDFS-6520](https://issues.apache.org/jira/browse/HDFS-6520) | hdfs fsck -move passes invalid length value when creating BlockReader | Major | . | Shengjun Xin | Xiao Chen | +| [HDFS-10267](https://issues.apache.org/jira/browse/HDFS-10267) | Extra "synchronized" on FsDatasetImpl#recoverAppend and FsDatasetImpl#recoverClose | Major | datanode | Colin P. McCabe | Colin P. McCabe | +| [YARN-4740](https://issues.apache.org/jira/browse/YARN-4740) | AM may not receive the container complete msg when it restarts | Major | . | sandflee | sandflee | +| [MAPREDUCE-6633](https://issues.apache.org/jira/browse/MAPREDUCE-6633) | AM should retry map attempts if the reduce task encounters commpression related errors. | Major | . | Rushabh S Shah | Rushabh S Shah | +| [YARN-4938](https://issues.apache.org/jira/browse/YARN-4938) | MiniYarnCluster should not request transitionToActive to RM on non-HA environment | Major | test | Akira Ajisaka | Eric Badger | +| [HADOOP-12406](https://issues.apache.org/jira/browse/HADOOP-12406) | AbstractMapWritable.readFields throws ClassNotFoundException with custom writables | Blocker | io | Nadeem Douba | Nadeem Douba | +| [HADOOP-12993](https://issues.apache.org/jira/browse/HADOOP-12993) | Change ShutdownHookManger complete shutdown log from INFO to DEBUG | Minor | . | Xiaoyu Yao | Xiaoyu Yao | +| [HDFS-10277](https://issues.apache.org/jira/browse/HDFS-10277) | PositionedReadable test testReadFullyZeroByteFile failing in HDFS | Major | test | Steve Loughran | Steve Loughran | +| [HDFS-10271](https://issues.apache.org/jira/browse/HDFS-10271) | Extra bytes are getting released from reservedSpace for append | Critical | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [HADOOP-12964](https://issues.apache.org/jira/browse/HADOOP-12964) | Http server vulnerable to clickjacking | Major | . | Haibo Chen | Haibo Chen | +| [YARN-4794](https://issues.apache.org/jira/browse/YARN-4794) | Deadlock in NMClientImpl | Critical | . | Sumana Sathish | Jian He | +| [HDFS-9772](https://issues.apache.org/jira/browse/HDFS-9772) | TestBlockReplacement#testThrottler doesn't work as expected | Minor | . | Yiqun Lin | Yiqun Lin | +| [HDFS-10270](https://issues.apache.org/jira/browse/HDFS-10270) | TestJMXGet:testNameNode() fails | Minor | test | Andras Bokor | Gergely Novák | +| [HDFS-10216](https://issues.apache.org/jira/browse/HDFS-10216) | distcp -diff relative path exception | Major | distcp | John Zhuge | Takashi Ohnishi | +| [YARN-4924](https://issues.apache.org/jira/browse/YARN-4924) | NM recovery race can lead to container not cleaned up | Major | nodemanager | Nathan Roberts | sandflee | +| [HADOOP-12989](https://issues.apache.org/jira/browse/HADOOP-12989) | Some tests in org.apache.hadoop.fs.shell.find occasionally time out | Major | test | Akira Ajisaka | Takashi Ohnishi | +| [HADOOP-13026](https://issues.apache.org/jira/browse/HADOOP-13026) | Should not wrap IOExceptions into a AuthenticationException in KerberosAuthenticator | Critical | . | Xuan Gong | Xuan Gong | +| [YARN-4940](https://issues.apache.org/jira/browse/YARN-4940) | yarn node -list -all failed if RM start with decommissioned node | Major | . | sandflee | sandflee | +| [YARN-4965](https://issues.apache.org/jira/browse/YARN-4965) | Distributed shell AM failed due to ClientHandlerException thrown by jersey | Critical | . | Sumana Sathish | Junping Du | +| [YARN-4934](https://issues.apache.org/jira/browse/YARN-4934) | Reserved Resource for QueueMetrics needs to be handled correctly in few cases | Major | capacity scheduler | Sunil G | Sunil G | +| [HDFS-10291](https://issues.apache.org/jira/browse/HDFS-10291) | TestShortCircuitLocalRead failing | Major | test | Steve Loughran | Steve Loughran | +| [HDFS-10275](https://issues.apache.org/jira/browse/HDFS-10275) | TestDataNodeMetrics failing intermittently due to TotalWriteTime counted incorrectly | Major | test | Yiqun Lin | Yiqun Lin | +| [MAPREDUCE-6649](https://issues.apache.org/jira/browse/MAPREDUCE-6649) | getFailureInfo not returning any failure info | Major | . | Eric Badger | Eric Badger | +| [HDFS-10265](https://issues.apache.org/jira/browse/HDFS-10265) | OEV tool fails to read edit xml file if OP\_UPDATE\_BLOCKS has no BLOCK tag | Minor | tools | Wan Chang | Wan Chang | +| [HDFS-9744](https://issues.apache.org/jira/browse/HDFS-9744) | TestDirectoryScanner#testThrottling occasionally time out after 300 seconds | Minor | datanode | Wei-Chiu Chuang | Yiqun Lin | +| [HDFS-10308](https://issues.apache.org/jira/browse/HDFS-10308) | TestRetryCacheWithHA#testRetryCacheOnStandbyNN failing | Major | test | Rakesh R | Rakesh R | +| [HDFS-10312](https://issues.apache.org/jira/browse/HDFS-10312) | Large block reports may fail to decode at NameNode due to 64 MB protobuf maximum length restriction. | Major | namenode | Chris Nauroth | Chris Nauroth | +| [MAPREDUCE-6680](https://issues.apache.org/jira/browse/MAPREDUCE-6680) | JHS UserLogDir scan algorithm sometime could skip directory with update in CloudFS (Azure FileSystem, S3, etc.) | Major | jobhistoryserver | Junping Du | Junping Du | +| [HDFS-9670](https://issues.apache.org/jira/browse/HDFS-9670) | DistCp throws NPE when source is root | Major | distcp | Yongjun Zhang | John Zhuge | +| [HADOOP-13042](https://issues.apache.org/jira/browse/HADOOP-13042) | Restore lost leveldbjni LICENSE and NOTICE changes | Major | . | Andrew Wang | Andrew Wang | +| [HADOOP-13043](https://issues.apache.org/jira/browse/HADOOP-13043) | Add LICENSE.txt entries for bundled javascript dependencies | Major | . | Andrew Wang | Andrew Wang | +| [HDFS-10319](https://issues.apache.org/jira/browse/HDFS-10319) | Balancer should not try to pair storages with different types | Minor | balancer & mover | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze | +| [HDFS-10309](https://issues.apache.org/jira/browse/HDFS-10309) | Balancer doesn't honor dfs.blocksize value defined with suffix k(kilo), m(mega), g(giga) | Minor | balancer & mover | Amit Anand | Amit Anand | +| [HDFS-9555](https://issues.apache.org/jira/browse/HDFS-9555) | LazyPersistFileScrubber should still sleep if there are errors in the clear progress | Major | . | Phil Yang | Phil Yang | +| [HADOOP-13052](https://issues.apache.org/jira/browse/HADOOP-13052) | ChecksumFileSystem mishandles crc file permissions | Major | fs | Daryn Sharp | Daryn Sharp | +| [HDFS-9905](https://issues.apache.org/jira/browse/HDFS-9905) | WebHdfsFileSystem#runWithRetry should display original stack trace on error | Major | test | Kihwal Lee | Wei-Chiu Chuang | +| [HADOOP-11418](https://issues.apache.org/jira/browse/HADOOP-11418) | Property "io.compression.codec.lzo.class" does not work with other value besides default | Major | io | fang fang chen | Yuanbo Liu | +| [HDFS-10318](https://issues.apache.org/jira/browse/HDFS-10318) | TestJMXGet hides the real error in case of test failure | Minor | test | Andras Bokor | Andras Bokor | +| [YARN-4556](https://issues.apache.org/jira/browse/YARN-4556) | TestFifoScheduler.testResourceOverCommit fails | Major | scheduler, test | Akihiro Suda | Akihiro Suda | +| [HDFS-10329](https://issues.apache.org/jira/browse/HDFS-10329) | Bad initialisation of StringBuffer in RequestHedgingProxyProvider.java | Minor | ha | Max Schaefer | Yiqun Lin | +| [HDFS-10313](https://issues.apache.org/jira/browse/HDFS-10313) | Distcp need to enforce the order of snapshot names passed to -diff | Major | distcp | Yongjun Zhang | Yiqun Lin | +| [MAPREDUCE-6199](https://issues.apache.org/jira/browse/MAPREDUCE-6199) | AbstractCounters are not reset completely on deserialization | Major | . | Anubhav Dhoot | Anubhav Dhoot | +| [HADOOP-13030](https://issues.apache.org/jira/browse/HADOOP-13030) | Handle special characters in passwords in KMS startup script | Major | kms | Xiao Chen | Xiao Chen | +| [YARN-4955](https://issues.apache.org/jira/browse/YARN-4955) | Add retry for SocketTimeoutException in TimelineClient | Critical | . | Xuan Gong | Xuan Gong | +| [HDFS-9958](https://issues.apache.org/jira/browse/HDFS-9958) | BlockManager#createLocatedBlocks can throw NPE for corruptBlocks on failed storages. | Major | . | Kuhu Shukla | Kuhu Shukla | +| [YARN-5008](https://issues.apache.org/jira/browse/YARN-5008) | LeveldbRMStateStore database can grow substantially leading to long recovery times | Major | resourcemanager | Jason Lowe | Jason Lowe | +| [YARN-5009](https://issues.apache.org/jira/browse/YARN-5009) | NMLeveldbStateStoreService database can grow substantially leading to longer recovery times | Major | nodemanager | Jason Lowe | Jason Lowe | +| [HADOOP-12378](https://issues.apache.org/jira/browse/HADOOP-12378) | Fix findbugs warnings in hadoop-tools module | Major | tools | Akira Ajisaka | Akira Ajisaka | +| [HDFS-10260](https://issues.apache.org/jira/browse/HDFS-10260) | TestFsDatasetImpl#testCleanShutdownOfVolume often fails | Major | datanode, test | Wei-Chiu Chuang | Rushabh S Shah | +| [HDFS-10335](https://issues.apache.org/jira/browse/HDFS-10335) | Mover$Processor#chooseTarget() always chooses the first matching target storage group | Critical | balancer & mover | Mingliang Liu | Mingliang Liu | +| [MAPREDUCE-6672](https://issues.apache.org/jira/browse/MAPREDUCE-6672) | TestTeraSort fails on Windows | Minor | test | Tibor Kiss | Tibor Kiss | +| [HDFS-10347](https://issues.apache.org/jira/browse/HDFS-10347) | Namenode report bad block method doesn't log the bad block or datanode. | Minor | namenode | Rushabh S Shah | Rushabh S Shah | +| [HADOOP-13072](https://issues.apache.org/jira/browse/HADOOP-13072) | WindowsGetSpaceUsed constructor should be public | Major | . | Vinayakumar B | Vinayakumar B | +| [MAPREDUCE-6537](https://issues.apache.org/jira/browse/MAPREDUCE-6537) | Include hadoop-pipes examples in the release tarball | Blocker | pipes | Allen Wittenauer | Kai Sasaki | +| [HDFS-10353](https://issues.apache.org/jira/browse/HDFS-10353) | Fix hadoop-hdfs-native-client compilation on Windows | Blocker | build | Brahma Reddy Battula | Brahma Reddy Battula | +| [HDFS-10344](https://issues.apache.org/jira/browse/HDFS-10344) | DistributedFileSystem#getTrashRoots should skip encryption zone that does not have .Trash | Major | . | Namit Maheshwari | Xiaoyu Yao | +| [HADOOP-13080](https://issues.apache.org/jira/browse/HADOOP-13080) | Refresh time in SysInfoWindows is in nanoseconds | Major | util | Inigo Goiri | Inigo Goiri | +| [YARN-4834](https://issues.apache.org/jira/browse/YARN-4834) | ProcfsBasedProcessTree doesn't track daemonized processes | Major | nodemanager | Nathan Roberts | Nathan Roberts | +| [HDFS-10320](https://issues.apache.org/jira/browse/HDFS-10320) | Rack failures may result in NN terminate | Major | . | Xiao Chen | Xiao Chen | +| [MAPREDUCE-6675](https://issues.apache.org/jira/browse/MAPREDUCE-6675) | TestJobImpl.testUnusableNode failed | Major | mrv2 | Haibo Chen | Haibo Chen | +| [YARN-4311](https://issues.apache.org/jira/browse/YARN-4311) | Removing nodes from include and exclude lists will not remove them from decommissioned nodes list | Major | . | Kuhu Shukla | Kuhu Shukla | +| [YARN-4984](https://issues.apache.org/jira/browse/YARN-4984) | LogAggregationService shouldn't swallow exception in handling createAppDir() which cause thread leak. | Critical | log-aggregation | Junping Du | Junping Du | +| [HADOOP-13098](https://issues.apache.org/jira/browse/HADOOP-13098) | Dynamic LogLevel setting page should accept case-insensitive log level string | Major | . | Junping Du | Junping Du | +| [HDFS-10324](https://issues.apache.org/jira/browse/HDFS-10324) | Trash directory in an encryption zone should be pre-created with correct permissions | Major | encryption | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [MAPREDUCE-6514](https://issues.apache.org/jira/browse/MAPREDUCE-6514) | Job hangs as ask is not updated after ramping down of all reducers | Blocker | applicationmaster | Varun Saxena | Varun Saxena | +| [HDFS-2043](https://issues.apache.org/jira/browse/HDFS-2043) | TestHFlush failing intermittently | Major | test | Aaron T. Myers | Yiqun Lin | +| [HADOOP-12751](https://issues.apache.org/jira/browse/HADOOP-12751) | While using kerberos Hadoop incorrectly assumes names with '@' to be non-simple | Critical | security | Bolke de Bruin | Bolke de Bruin | +| [MAPREDUCE-6689](https://issues.apache.org/jira/browse/MAPREDUCE-6689) | MapReduce job can infinitely increase number of reducer resource requests | Blocker | . | Wangda Tan | Wangda Tan | +| [YARN-4747](https://issues.apache.org/jira/browse/YARN-4747) | AHS error 500 due to NPE when container start event is missing | Major | timelineserver | Jason Lowe | Varun Saxena | +| [HDFS-9939](https://issues.apache.org/jira/browse/HDFS-9939) | Increase DecompressorStream skip buffer size | Major | . | Yongjun Zhang | John Zhuge | +| [YARN-5048](https://issues.apache.org/jira/browse/YARN-5048) | DelegationTokenRenewer#skipTokenRenewal may throw NPE | Major | . | Jian He | Jian He | +| [YARN-4926](https://issues.apache.org/jira/browse/YARN-4926) | Change nodelabel rest API invalid reponse status to 400 | Major | . | Bibin A Chundatt | Bibin A Chundatt | +| [HDFS-10372](https://issues.apache.org/jira/browse/HDFS-10372) | Fix for failing TestFsDatasetImpl#testCleanShutdownOfVolume | Major | test | Rushabh S Shah | Rushabh S Shah | +| [MAPREDUCE-6684](https://issues.apache.org/jira/browse/MAPREDUCE-6684) | High contention on scanning of user directory under immediate\_done in Job History Server | Critical | jobhistoryserver | Haibo Chen | Haibo Chen | +| [HDFS-6187](https://issues.apache.org/jira/browse/HDFS-6187) | Update the document of hftp/hsftp in branch-2 to mention that they are deprecated | Major | . | Haohui Mai | Gergely Novák | +| [YARN-4768](https://issues.apache.org/jira/browse/YARN-4768) | getAvailablePhysicalMemorySize can be inaccurate on linux | Major | nodemanager | Nathan Roberts | Nathan Roberts | +| [HADOOP-13125](https://issues.apache.org/jira/browse/HADOOP-13125) | FS Contract tests don't report FS initialization errors well | Minor | test | Steve Loughran | Steve Loughran | +| [YARN-5029](https://issues.apache.org/jira/browse/YARN-5029) | RM needs to send update event with YarnApplicationState as Running to ATS/AHS | Critical | . | Xuan Gong | Xuan Gong | +| [HADOOP-13116](https://issues.apache.org/jira/browse/HADOOP-13116) | Jets3tNativeS3FileSystemContractTest does not run. | Minor | test | Chris Nauroth | Chris Nauroth | +| [MAPREDUCE-6639](https://issues.apache.org/jira/browse/MAPREDUCE-6639) | Process hangs in LocatedFileStatusFetcher if FileSystem.get throws | Major | mrv2 | Ryan Blue | Ryan Blue | +| [HADOOP-11180](https://issues.apache.org/jira/browse/HADOOP-11180) | Change log message "token.Token: Cannot find class for token kind kms-dt" to debug | Major | kms, security | Yi Liu | Yi Liu | +| [MAPREDUCE-6558](https://issues.apache.org/jira/browse/MAPREDUCE-6558) | multibyte delimiters with compressed input files generate duplicate records | Major | mrv1, mrv2 | Wilfred Spiegelenburg | Wilfred Spiegelenburg | +| [HADOOP-13083](https://issues.apache.org/jira/browse/HADOOP-13083) | The number of javadocs warnings is limited to 100 | Critical | . | Li Lu | Gergely Novák | +| [MAPREDUCE-6513](https://issues.apache.org/jira/browse/MAPREDUCE-6513) | MR job got hanged forever when one NM unstable for some time | Critical | applicationmaster, resourcemanager | Bob.zhao | Varun Saxena | +| [HDFS-10333](https://issues.apache.org/jira/browse/HDFS-10333) | Intermittent org.apache.hadoop.hdfs.TestFileAppend failure in trunk | Major | hdfs | Yongjun Zhang | Yiqun Lin | +| [HADOOP-12942](https://issues.apache.org/jira/browse/HADOOP-12942) | hadoop credential commands non-obviously use password of "none" | Major | security | Mike Yoder | Mike Yoder | +| [YARN-4325](https://issues.apache.org/jira/browse/YARN-4325) | Nodemanager log handlers fail to send finished/failed events in some cases | Critical | . | Junping Du | Junping Du | +| [HDFS-10242](https://issues.apache.org/jira/browse/HDFS-10242) | Cannot create space quota of zero | Major | documentation, fs | Takashi Ohnishi | Takashi Ohnishi | +| [MAPREDUCE-6693](https://issues.apache.org/jira/browse/MAPREDUCE-6693) | ArrayIndexOutOfBoundsException occurs when the length of the job name is equal to mapreduce.jobhistory.jobname.limit | Critical | . | Bibin A Chundatt | Ajith S | +| [HADOOP-13163](https://issues.apache.org/jira/browse/HADOOP-13163) | Reuse pre-computed filestatus in Distcp-CopyMapper | Minor | tools/distcp | Rajesh Balamohan | Rajesh Balamohan | +| [HDFS-10303](https://issues.apache.org/jira/browse/HDFS-10303) | DataStreamer#ResponseProcessor calculates packet ack latency incorrectly. | Major | hdfs-client | Surendra Singh Lilhore | Surendra Singh Lilhore | +| [MAPREDUCE-6698](https://issues.apache.org/jira/browse/MAPREDUCE-6698) | Increase timeout on TestUnnecessaryBlockingOnHistoryFileInfo.testTwoThreadsQueryingDifferentJobOfSameUser | Major | jobhistoryserver | Haibo Chen | Haibo Chen | +| [HADOOP-13159](https://issues.apache.org/jira/browse/HADOOP-13159) | Fix potential NPE in Metrics2 source for DecayRpcScheduler | Major | ipc | Xiaoyu Yao | Xiaoyu Yao | +| [HDFS-10397](https://issues.apache.org/jira/browse/HDFS-10397) | Distcp should ignore -delete option if -diff option is provided instead of exiting | Major | distcp | Mingliang Liu | Mingliang Liu | +| [HDFS-10381](https://issues.apache.org/jira/browse/HDFS-10381) | DataStreamer DataNode exclusion log message should be warning | Minor | hdfs-client | John Zhuge | John Zhuge | +| [HDFS-9226](https://issues.apache.org/jira/browse/HDFS-9226) | MiniDFSCluster leaks dependency Mockito via DataNodeTestUtils | Major | test | Josh Elser | Josh Elser | +| [HADOOP-13138](https://issues.apache.org/jira/browse/HADOOP-13138) | Unable to append to a SequenceFile with Compression.NONE. | Critical | . | Gervais Mickaël | Vinayakumar B | +| [YARN-4925](https://issues.apache.org/jira/browse/YARN-4925) | ContainerRequest in AMRMClient, application should be able to specify nodes/racks together with nodeLabelExpression | Major | . | Bibin A Chundatt | Bibin A Chundatt | +| [HADOOP-13157](https://issues.apache.org/jira/browse/HADOOP-13157) | Follow-on improvements to hadoop credential commands | Major | security | Mike Yoder | Mike Yoder | +| [YARN-3840](https://issues.apache.org/jira/browse/YARN-3840) | Resource Manager web ui issue when sorting application by id (with application having id \> 9999) | Major | resourcemanager | LINTE | Varun Saxena | +| [HADOOP-13177](https://issues.apache.org/jira/browse/HADOOP-13177) | Native tests fail on OS X, because DYLD\_LIBRARY\_PATH is not defined to include libhadoop.dylib. | Minor | build | Chris Nauroth | Chris Nauroth | +| [HADOOP-12767](https://issues.apache.org/jira/browse/HADOOP-12767) | update apache httpclient version to 4.5.2; httpcore to 4.4.4 | Major | build | Artem Aliev | Artem Aliev | +| [YARN-5100](https://issues.apache.org/jira/browse/YARN-5100) | The YarnApplicationState is always running in ATS no matter the application is running or finishes. | Blocker | . | Xuan Gong | Xuan Gong | +| [HADOOP-13183](https://issues.apache.org/jira/browse/HADOOP-13183) | S3A proxy tests fail after httpclient/httpcore upgrade. | Major | fs/s3 | Chris Nauroth | Steve Loughran | +| [YARN-5020](https://issues.apache.org/jira/browse/YARN-5020) | Fix Documentation for Yarn Capacity Scheduler on Resource Calculator | Minor | . | Jo Desmet | Takashi Ohnishi | +| [HDFS-10424](https://issues.apache.org/jira/browse/HDFS-10424) | DatanodeLifelineProtocol not able to use under security cluster | Blocker | . | gu-chi | Chris Nauroth | +| [HDFS-10438](https://issues.apache.org/jira/browse/HDFS-10438) | When NameNode HA is configured to use the lifeline RPC server, it should log the address of that server. | Minor | ha, namenode | KWON BYUNGCHANG | Chris Nauroth | +| [HDFS-10439](https://issues.apache.org/jira/browse/HDFS-10439) | Update setOwner doc in HdfsPermissionsGuide | Minor | documentation | John Zhuge | John Zhuge | +| [MAPREDUCE-6607](https://issues.apache.org/jira/browse/MAPREDUCE-6607) | Enable regex pattern matching when mapreduce.task.files.preserve.filepattern is set | Minor | applicationmaster | Maysam Yabandeh | Kai Sasaki | +| [YARN-5103](https://issues.apache.org/jira/browse/YARN-5103) | With NM recovery enabled, restarting NM multiple times results in AM restart | Critical | yarn | Sumana Sathish | Junping Du | +| [YARN-5055](https://issues.apache.org/jira/browse/YARN-5055) | max apps per user can be larger than max per queue | Minor | capacityscheduler, resourcemanager | Jason Lowe | Eric Badger | +| [YARN-3971](https://issues.apache.org/jira/browse/YARN-3971) | Skip RMNodeLabelsManager#checkRemoveFromClusterNodeLabelsOfQueue on nodelabel recovery | Critical | resourcemanager | Bibin A Chundatt | Bibin A Chundatt | +| [HDFS-9365](https://issues.apache.org/jira/browse/HDFS-9365) | Balancer does not work with the HDFS-6376 HA setup | Major | balancer & mover | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze | +| [YARN-3344](https://issues.apache.org/jira/browse/YARN-3344) | Fix warning - procfs stat file is not in the expected format | Major | . | Jon Bringhurst | Ravindra Kumar Naik | +| [YARN-4459](https://issues.apache.org/jira/browse/YARN-4459) | container-executor should only kill process groups | Major | nodemanager | Jun Gong | Jun Gong | +| [YARN-5166](https://issues.apache.org/jira/browse/YARN-5166) | javadoc:javadoc goal fails on hadoop-yarn-client | Major | . | Andras Bokor | Andras Bokor | +| [HDFS-10276](https://issues.apache.org/jira/browse/HDFS-10276) | HDFS should not expose path info that user has no permission to see. | Major | . | Kevin Cox | Yuanbo Liu | +| [YARN-5132](https://issues.apache.org/jira/browse/YARN-5132) | Exclude generated protobuf sources from YARN Javadoc build | Critical | . | Subru Krishnan | Subru Krishnan | +| [HADOOP-13132](https://issues.apache.org/jira/browse/HADOOP-13132) | Handle ClassCastException on AuthenticationException in LoadBalancingKMSClientProvider | Major | kms | Miklos Szurap | Wei-Chiu Chuang | +| [HDFS-10415](https://issues.apache.org/jira/browse/HDFS-10415) | TestDistributedFileSystem#MyDistributedFileSystem attempts to set up statistics before initialize() is called | Major | test | Sangjin Lee | Mingliang Liu | +| [HADOOP-13137](https://issues.apache.org/jira/browse/HADOOP-13137) | TraceAdmin should support Kerberized cluster | Major | tracing | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HDFS-9476](https://issues.apache.org/jira/browse/HDFS-9476) | TestDFSUpgradeFromImage#testUpgradeFromRel1BBWImage occasionally fail | Major | . | Wei-Chiu Chuang | Masatake Iwasaki | +| [HDFS-10367](https://issues.apache.org/jira/browse/HDFS-10367) | TestDFSShell.testMoveWithTargetPortEmpty fails with Address bind exception. | Major | test | Brahma Reddy Battula | Brahma Reddy Battula | +| [HDFS-10471](https://issues.apache.org/jira/browse/HDFS-10471) | DFSAdmin#SetQuotaCommand's help msg is not correct | Minor | documentation | Yiqun Lin | Yiqun Lin | +| [YARN-5098](https://issues.apache.org/jira/browse/YARN-5098) | Yarn Application log Aggreagation fails due to NM can not get correct HDFS delegation token | Major | yarn | Yesha Vora | Jian He | +| [HADOOP-13155](https://issues.apache.org/jira/browse/HADOOP-13155) | Implement TokenRenewer to renew and cancel delegation tokens in KMS | Major | kms, security | Xiao Chen | Xiao Chen | +| [HDFS-10481](https://issues.apache.org/jira/browse/HDFS-10481) | HTTPFS server should correctly impersonate as end user to open file | Major | httpfs | Xiao Chen | Xiao Chen | +| [HDFS-10485](https://issues.apache.org/jira/browse/HDFS-10485) | Fix findbugs warning in FSEditLog.java | Major | . | Akira Ajisaka | Akira Ajisaka | +| [HDFS-10458](https://issues.apache.org/jira/browse/HDFS-10458) | getFileEncryptionInfo should return quickly for non-encrypted cluster | Major | encryption, namenode | Zhe Zhang | Zhe Zhang | +| [YARN-5206](https://issues.apache.org/jira/browse/YARN-5206) | RegistrySecurity includes id:pass in exception text if considered invalid | Minor | client, security | Steve Loughran | Steve Loughran | +| [HDFS-10220](https://issues.apache.org/jira/browse/HDFS-10220) | A large number of expired leases can make namenode unresponsive and cause failover | Major | namenode | Nicolas Fraison | Nicolas Fraison | +| [HADOOP-13249](https://issues.apache.org/jira/browse/HADOOP-13249) | RetryInvocationHandler need wrap InterruptedException in IOException when call Thread.sleep | Major | ipc | zhihai xu | zhihai xu | +| [HADOOP-13213](https://issues.apache.org/jira/browse/HADOOP-13213) | Small Documentation bug with AuthenticatedURL in hadoop-auth | Minor | documentation | Tom Ellis | Tom Ellis | +| [HADOOP-13079](https://issues.apache.org/jira/browse/HADOOP-13079) | Add -q option to Ls to print ? instead of non-printable characters | Major | . | John Zhuge | John Zhuge | +| [HDFS-10516](https://issues.apache.org/jira/browse/HDFS-10516) | Fix bug when warming up EDEK cache of more than one encryption zone | Major | encryption, namenode | Xiao Chen | Xiao Chen | +| [HADOOP-13270](https://issues.apache.org/jira/browse/HADOOP-13270) | BZip2CompressionInputStream finds the same compression marker twice in corner case, causing duplicate data blocks | Critical | . | Haibo Chen | Kai Sasaki | +| [HADOOP-13179](https://issues.apache.org/jira/browse/HADOOP-13179) | GenericOptionsParser is not thread-safe because commons-cli OptionBuilder is not thread-safe | Minor | . | hongbin ma | hongbin ma | +| [HADOOP-13244](https://issues.apache.org/jira/browse/HADOOP-13244) | o.a.h.ipc.Server#Server should honor handlerCount when queueSizePerHandler is specified in consturctor | Minor | ipc | Xiaoyu Yao | Kai Sasaki | +| [HADOOP-13245](https://issues.apache.org/jira/browse/HADOOP-13245) | Fix up some misc create-release issues | Blocker | build | Allen Wittenauer | Allen Wittenauer | +| [HDFS-10505](https://issues.apache.org/jira/browse/HDFS-10505) | OIV's ReverseXML processor should support ACLs | Major | tools | Colin P. McCabe | Surendra Singh Lilhore | +| [HDFS-10525](https://issues.apache.org/jira/browse/HDFS-10525) | Fix NPE in CacheReplicationMonitor#rescanCachedBlockMap | Major | caching | Xiao Chen | Xiao Chen | +| [YARN-5237](https://issues.apache.org/jira/browse/YARN-5237) | Fix missing log files issue in rolling log aggregation. | Major | . | Siddharth Seth | Xuan Gong | +| [HDFS-10532](https://issues.apache.org/jira/browse/HDFS-10532) | Typo in RollingUpgrade docs | Major | documentation | Arpit Agarwal | Yiqun Lin | +| [HADOOP-3733](https://issues.apache.org/jira/browse/HADOOP-3733) | "s3:" URLs break when Secret Key contains a slash, even if encoded | Minor | fs/s3 | Stuart Sierra | Steve Loughran | +| [HDFS-9466](https://issues.apache.org/jira/browse/HDFS-9466) | TestShortCircuitCache#testDataXceiverCleansUpSlotsOnFailure is flaky | Major | fs, hdfs-client | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HADOOP-13255](https://issues.apache.org/jira/browse/HADOOP-13255) | KMSClientProvider should check and renew tgt when doing delegation token operations. | Major | kms | Xiao Chen | Xiao Chen | +| [HADOOP-13285](https://issues.apache.org/jira/browse/HADOOP-13285) | DecayRpcScheduler MXBean should only report decayed CallVolumeSummary | Major | ipc | Namit Maheshwari | Xiaoyu Yao | +| [HADOOP-13149](https://issues.apache.org/jira/browse/HADOOP-13149) | Windows distro build fails on dist-copynativelibs. | Blocker | build | Chris Nauroth | Chris Nauroth | +| [YARN-5246](https://issues.apache.org/jira/browse/YARN-5246) | NMWebAppFilter web redirects drop query parameters | Major | . | Varun Vasudev | Varun Vasudev | +| [HDFS-10474](https://issues.apache.org/jira/browse/HDFS-10474) | hftp copy fails when file name with Chinese+special char in branch-2 | Critical | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [HADOOP-13192](https://issues.apache.org/jira/browse/HADOOP-13192) | org.apache.hadoop.util.LineReader cannot handle multibyte delimiters correctly | Critical | util | binde | binde | +| [HDFS-10448](https://issues.apache.org/jira/browse/HDFS-10448) | CacheManager#addInternal tracks bytesNeeded incorrectly when dealing with replication factors other than 1 | Major | caching | Yiqun Lin | Yiqun Lin | +| [YARN-5197](https://issues.apache.org/jira/browse/YARN-5197) | RM leaks containers if running container disappears from node update | Major | resourcemanager | Jason Lowe | Jason Lowe | +| [HADOOP-13287](https://issues.apache.org/jira/browse/HADOOP-13287) | TestS3ACredentials#testInstantiateFromURL fails if AWS secret key contains '+'. | Minor | fs/s3, test | Chris Nauroth | Chris Nauroth | +| [HDFS-10556](https://issues.apache.org/jira/browse/HDFS-10556) | DistCpOptions should be validated automatically | Major | distcp | Mingliang Liu | Mingliang Liu | +| [MAPREDUCE-6725](https://issues.apache.org/jira/browse/MAPREDUCE-6725) | Javadoc for CLI#listEvents() contains no-existent param | Minor | client, documentation | Shen Yinjie | Shen Yinjie | +| [HDFS-7959](https://issues.apache.org/jira/browse/HDFS-7959) | WebHdfs logging is missing on Datanode | Critical | . | Kihwal Lee | Kihwal Lee | +| [MAPREDUCE-6542](https://issues.apache.org/jira/browse/MAPREDUCE-6542) | HistoryViewer uses SimpleDateFormat, but SimpleDateFormat is not threadsafe | Major | jobhistoryserver | zhangyubiao | zhangyubiao | +| [HADOOP-13251](https://issues.apache.org/jira/browse/HADOOP-13251) | Authenticate with Kerberos credentials when renewing KMS delegation token | Major | kms | Xiao Chen | Xiao Chen | +| [HADOOP-13316](https://issues.apache.org/jira/browse/HADOOP-13316) | Enforce Kerberos authentication for required ops in DelegationTokenAuthenticator | Blocker | kms, security | Xiao Chen | Xiao Chen | +| [HDFS-9852](https://issues.apache.org/jira/browse/HDFS-9852) | hdfs dfs -setfacl error message is misleading | Minor | . | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [YARN-5262](https://issues.apache.org/jira/browse/YARN-5262) | Optimize sending RMNodeFinishedContainersPulledByAMEvent for every AM heartbeat | Major | resourcemanager | Rohith Sharma K S | Rohith Sharma K S | +| [HADOOP-12345](https://issues.apache.org/jira/browse/HADOOP-12345) | Pad hostname correctly in CredentialsSys.java | Critical | nfs | Pradeep Nayak Udupi Kadbet | Pradeep Nayak Udupi Kadbet | +| [HADOOP-13314](https://issues.apache.org/jira/browse/HADOOP-13314) | Remove 'package-info.java' from 'test\java\org\apache\hadoop\fs\shell\' to remove eclipse compile error | Trivial | . | Vinayakumar B | Vinayakumar B | +| [HDFS-10589](https://issues.apache.org/jira/browse/HDFS-10589) | Javadoc for HAState#HAState and HAState#setStateInternal contains non-existent params | Minor | documentation, hdfs | Shen Yinjie | Shen Yinjie | +| [YARN-5286](https://issues.apache.org/jira/browse/YARN-5286) | Add RPC port info in RM web service's response when getting app status | Major | . | Jun Gong | Jun Gong | +| [YARN-5214](https://issues.apache.org/jira/browse/YARN-5214) | Pending on synchronized method DirectoryCollection#checkDirs can hang NM's NodeStatusUpdater | Critical | nodemanager | Junping Du | Junping Du | +| [HADOOP-13350](https://issues.apache.org/jira/browse/HADOOP-13350) | Additional fix to LICENSE and NOTICE | Blocker | build | Xiao Chen | Xiao Chen | +| [HDFS-10592](https://issues.apache.org/jira/browse/HDFS-10592) | Fix intermittent test failure of TestNameNodeResourceChecker#testCheckThatNameNodeResourceMonitorIsRunning | Major | test | Rakesh R | Rakesh R | +| [HADOOP-13320](https://issues.apache.org/jira/browse/HADOOP-13320) | Fix arguments check in documentation for WordCount v2.0 | Minor | documentation | niccolo becchi | niccolo becchi | +| [YARN-5314](https://issues.apache.org/jira/browse/YARN-5314) | ConcurrentModificationException in ATS v1.5 EntityGroupFSTimelineStore | Major | timelineserver | Karam Singh | Li Lu | +| [YARN-4939](https://issues.apache.org/jira/browse/YARN-4939) | the decommissioning Node should keep alive if NM restart | Major | . | sandflee | sandflee | +| [HADOOP-12893](https://issues.apache.org/jira/browse/HADOOP-12893) | Verify LICENSE.txt and NOTICE.txt | Blocker | build | Allen Wittenauer | Xiao Chen | +| [HADOOP-13352](https://issues.apache.org/jira/browse/HADOOP-13352) | Make X-FRAME-OPTIONS configurable in HttpServer2 | Major | net, security | Anu Engineer | Anu Engineer | +| [HDFS-10336](https://issues.apache.org/jira/browse/HDFS-10336) | TestBalancer failing intermittently because of not reseting UserGroupInformation completely | Major | test | Yiqun Lin | Yiqun Lin | +| [HDFS-10512](https://issues.apache.org/jira/browse/HDFS-10512) | VolumeScanner may terminate due to NPE in DataNode.reportBadBlocks | Major | datanode | Wei-Chiu Chuang | Yiqun Lin | +| [YARN-5337](https://issues.apache.org/jira/browse/YARN-5337) | Fix OOM issue in DistributedShell. AM failed with "java.lang.OutOfMemoryError: GC overhead limit exceeded" | Major | . | Sumana Sathish | Jian He | +| [HADOOP-13297](https://issues.apache.org/jira/browse/HADOOP-13297) | Add missing dependency in setting maven-remote-resource-plugin to fix builds | Major | build | Akira Ajisaka | Sean Busbey | +| [YARN-5270](https://issues.apache.org/jira/browse/YARN-5270) | Solve miscellaneous issues caused by YARN-4844 | Blocker | . | Wangda Tan | Wangda Tan | +| [HDFS-10579](https://issues.apache.org/jira/browse/HDFS-10579) | HDFS web interfaces lack configs for X-FRAME-OPTIONS protection | Major | datanode, namenode | Anu Engineer | Anu Engineer | +| [MAPREDUCE-6625](https://issues.apache.org/jira/browse/MAPREDUCE-6625) | TestCLI#testGetJob fails occasionally | Major | test | Jason Lowe | Haibo Chen | +| [HADOOP-13315](https://issues.apache.org/jira/browse/HADOOP-13315) | FileContext#umask is not initialized properly | Minor | . | John Zhuge | John Zhuge | +| [YARN-5353](https://issues.apache.org/jira/browse/YARN-5353) | ResourceManager can leak delegation tokens when they are shared across apps | Critical | resourcemanager | Jason Lowe | Jason Lowe | +| [HADOOP-11361](https://issues.apache.org/jira/browse/HADOOP-11361) | Fix a race condition in MetricsSourceAdapter.updateJmxCache | Major | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [HDFS-10544](https://issues.apache.org/jira/browse/HDFS-10544) | Balancer doesn't work with IPFailoverProxyProvider | Major | balancer & mover, ha | Zhe Zhang | Zhe Zhang | +| [HADOOP-13351](https://issues.apache.org/jira/browse/HADOOP-13351) | TestDFSClientSocketSize buffer size tests are flaky | Major | . | Aaron Fabbri | Aaron Fabbri | +| [HADOOP-13202](https://issues.apache.org/jira/browse/HADOOP-13202) | Avoid possible overflow in org.apache.hadoop.util.bloom.BloomFilter#getNBytes | Major | util | zhengbing li | Kai Sasaki | +| [HDFS-10603](https://issues.apache.org/jira/browse/HDFS-10603) | Fix flaky tests in org.apache.hadoop.hdfs.server.namenode.snapshot.TestOpenFilesWithSnapshot | Major | hdfs, namenode | Yongjun Zhang | Yiqun Lin | +| [HADOOP-12991](https://issues.apache.org/jira/browse/HADOOP-12991) | Conflicting default ports in DelegateToFileSystem | Major | fs | Kevin Hogeland | Kai Sasaki | +| [YARN-5309](https://issues.apache.org/jira/browse/YARN-5309) | Fix SSLFactory truststore reloader thread leak in TimelineClientImpl | Blocker | timelineserver, yarn | Thomas Friedrich | Weiwei Yang | +| [HADOOP-13387](https://issues.apache.org/jira/browse/HADOOP-13387) | users always get told off for using S3 —even when not using it. | Minor | fs/s3 | Steve Loughran | Steve Loughran | +| [YARN-5340](https://issues.apache.org/jira/browse/YARN-5340) | Race condition in RollingLevelDBTimelineStore#getAndSetStartTime() | Critical | timelineserver | Sumana Sathish | Li Lu | +| [HDFS-8914](https://issues.apache.org/jira/browse/HDFS-8914) | Document HA support in the HDFS HdfsDesign.md | Major | documentation | Ravindra Babu | Lars Francke | +| [HADOOP-12588](https://issues.apache.org/jira/browse/HADOOP-12588) | Fix intermittent test failure of TestGangliaMetrics | Major | . | Tsuyoshi Ozawa | Masatake Iwasaki | +| [HADOOP-13240](https://issues.apache.org/jira/browse/HADOOP-13240) | TestAclCommands.testSetfaclValidations fail | Minor | test | linbao111 | John Zhuge | +| [HADOOP-13389](https://issues.apache.org/jira/browse/HADOOP-13389) | TestS3ATemporaryCredentials.testSTS error when using IAM credentials | Major | fs/s3 | Steven K. Wong | Steven K. Wong | +| [HADOOP-13406](https://issues.apache.org/jira/browse/HADOOP-13406) | S3AFileSystem: Consider reusing filestatus in delete() and mkdirs() | Minor | fs/s3 | Rajesh Balamohan | Rajesh Balamohan | +| [MAPREDUCE-6744](https://issues.apache.org/jira/browse/MAPREDUCE-6744) | Increase timeout on TestDFSIO tests | Major | . | Eric Badger | Eric Badger | +| [HDFS-10688](https://issues.apache.org/jira/browse/HDFS-10688) | BPServiceActor may run into a tight loop for sending block report when hitting IOException | Major | datanode | Jing Zhao | Chen Liang | +| [HDFS-10671](https://issues.apache.org/jira/browse/HDFS-10671) | Fix typo in HdfsRollingUpgrade.md | Trivial | documentation | Yiqun Lin | Yiqun Lin | +| [HADOOP-13422](https://issues.apache.org/jira/browse/HADOOP-13422) | ZKDelegationTokenSecretManager JaasConfig does not work well with other ZK users in process | Major | . | Sergey Shelukhin | Sergey Shelukhin | +| [HDFS-10696](https://issues.apache.org/jira/browse/HDFS-10696) | TestHDFSCLI fails | Major | test | Akira Ajisaka | Kai Sasaki | +| [YARN-5432](https://issues.apache.org/jira/browse/YARN-5432) | Lock already held by another process while LevelDB cache store creation for dag | Critical | timelineserver | Karam Singh | Li Lu | +| [YARN-5438](https://issues.apache.org/jira/browse/YARN-5438) | TimelineClientImpl leaking FileSystem Instances causing Long running services like HiverServer2 daemon going OOM | Major | timelineserver | Karam Singh | Rohith Sharma K S | +| [HADOOP-13381](https://issues.apache.org/jira/browse/HADOOP-13381) | KMS clients should use KMS Delegation Tokens from current UGI. | Critical | kms | Xiao Chen | Xiao Chen | +| [HDFS-10691](https://issues.apache.org/jira/browse/HDFS-10691) | FileDistribution fails in hdfs oiv command due to ArrayIndexOutOfBoundsException | Major | . | Yiqun Lin | Yiqun Lin | +| [YARN-5121](https://issues.apache.org/jira/browse/YARN-5121) | fix some container-executor portability issues | Blocker | nodemanager, security | Allen Wittenauer | Allen Wittenauer | +| [MAPREDUCE-6724](https://issues.apache.org/jira/browse/MAPREDUCE-6724) | Single shuffle to memory must not exceed Integer#MAX\_VALUE | Major | mrv2 | Haibo Chen | Haibo Chen | +| [HDFS-5805](https://issues.apache.org/jira/browse/HDFS-5805) | TestCheckpoint.testCheckpoint fails intermittently on branch2 | Major | . | Mit Desai | Eric Badger | +| [HADOOP-13459](https://issues.apache.org/jira/browse/HADOOP-13459) | hadoop-azure runs several test cases repeatedly, causing unnecessarily long running time. | Minor | fs/azure, test | Chris Nauroth | Chris Nauroth | +| [HDFS-742](https://issues.apache.org/jira/browse/HDFS-742) | A down DataNode makes Balancer to hang on repeatingly asking NameNode its partial block list | Minor | balancer & mover | Hairong Kuang | Mit Desai | +| [YARN-4280](https://issues.apache.org/jira/browse/YARN-4280) | CapacityScheduler reservations may not prevent indefinite postponement on a busy cluster | Major | capacity scheduler | Kuhu Shukla | Kuhu Shukla | +| [YARN-5462](https://issues.apache.org/jira/browse/YARN-5462) | TestNodeStatusUpdater.testNodeStatusUpdaterRetryAndNMShutdown fails intermittently | Major | . | Eric Badger | Eric Badger | +| [HDFS-10710](https://issues.apache.org/jira/browse/HDFS-10710) | In BlockManager#rescanPostponedMisreplicatedBlocks(), postponed misreplicated block counts should be retrieved with NN lock protection | Major | namenode | Rui Gao | Rui Gao | +| [YARN-5469](https://issues.apache.org/jira/browse/YARN-5469) | Increase timeout of TestAmFilter.testFilter | Minor | . | Eric Badger | Eric Badger | +| [HDFS-10569](https://issues.apache.org/jira/browse/HDFS-10569) | A bug causes OutOfIndex error in BlockListAsLongs | Minor | . | Weiwei Yang | Weiwei Yang | +| [HADOOP-13434](https://issues.apache.org/jira/browse/HADOOP-13434) | Add quoting to Shell class | Major | . | Owen O'Malley | Owen O'Malley | +| [MAPREDUCE-6682](https://issues.apache.org/jira/browse/MAPREDUCE-6682) | TestMRCJCFileOutputCommitter fails intermittently | Major | test | Brahma Reddy Battula | Akira Ajisaka | +| [HDFS-10716](https://issues.apache.org/jira/browse/HDFS-10716) | In Balancer, the target task should be removed when its size \< 0. | Minor | balancer & mover | Yiqun Lin | Yiqun Lin | +| [HDFS-10722](https://issues.apache.org/jira/browse/HDFS-10722) | Fix race condition in TestEditLog#testBatchedSyncWithClosedLogs | Major | hdfs | Daryn Sharp | Daryn Sharp | +| [HADOOP-13467](https://issues.apache.org/jira/browse/HADOOP-13467) | Shell#getSignalKillCommand should use the bash builtin on Linux | Major | . | Arpit Agarwal | Arpit Agarwal | +| [HDFS-10717](https://issues.apache.org/jira/browse/HDFS-10717) | Fix findbugs warnings of hadoop-hdfs-client in branch-2 | Major | . | Akira Ajisaka | Akira Ajisaka | +| [HDFS-10343](https://issues.apache.org/jira/browse/HDFS-10343) | BlockManager#createLocatedBlocks may return blocks on failed storages | Major | hdfs | Daryn Sharp | Kuhu Shukla | +| [HDFS-10715](https://issues.apache.org/jira/browse/HDFS-10715) | NPE when applying AvailableSpaceBlockPlacementPolicy | Major | namenode | Guangbin Zhu | Guangbin Zhu | +| [YARN-4624](https://issues.apache.org/jira/browse/YARN-4624) | NPE in PartitionQueueCapacitiesInfo while accessing Schduler UI | Major | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [HADOOP-10823](https://issues.apache.org/jira/browse/HADOOP-10823) | TestReloadingX509TrustManager is flaky | Major | . | Ratandeep Ratti | Mingliang Liu | +| [HADOOP-13457](https://issues.apache.org/jira/browse/HADOOP-13457) | Remove hardcoded absolute path for shell executable | Major | util | Arpit Agarwal | Chen Liang | +| [HADOOP-13439](https://issues.apache.org/jira/browse/HADOOP-13439) | Fix race between TestMetricsSystemImpl and TestGangliaMetrics | Minor | test | Masatake Iwasaki | Chen Liang | +| [MAPREDUCE-6750](https://issues.apache.org/jira/browse/MAPREDUCE-6750) | TestHSAdminServer.testRefreshSuperUserGroups is failing | Minor | test | Kihwal Lee | Kihwal Lee | +| [HADOOP-13473](https://issues.apache.org/jira/browse/HADOOP-13473) | Tracing in IPC Server is broken | Major | . | Wei-Chiu Chuang | Daryn Sharp | +| [HDFS-10738](https://issues.apache.org/jira/browse/HDFS-10738) | Fix TestRefreshUserMappings.testRefreshSuperUserGroupsConfiguration test failure | Major | test | Rakesh R | Rakesh R | +| [HADOOP-13299](https://issues.apache.org/jira/browse/HADOOP-13299) | JMXJsonServlet is vulnerable to TRACE | Minor | . | Haibo Chen | Haibo Chen | +| [HDFS-8224](https://issues.apache.org/jira/browse/HDFS-8224) | Schedule a block for scanning if its metadata file is corrupt | Major | datanode | Rushabh S Shah | Rushabh S Shah | +| [YARN-5382](https://issues.apache.org/jira/browse/YARN-5382) | RM does not audit log kill request for active applications | Major | resourcemanager | Jason Lowe | Vrushali C | +| [HDFS-10643](https://issues.apache.org/jira/browse/HDFS-10643) | Namenode should use loginUser(hdfs) to generateEncryptedKey | Major | encryption, namenode | Xiaoyu Yao | Xiaoyu Yao | +| [HDFS-8897](https://issues.apache.org/jira/browse/HDFS-8897) | Balancer should handle fs.defaultFS trailing slash in HA | Major | balancer & mover | LINTE | John Zhuge | +| [HDFS-10731](https://issues.apache.org/jira/browse/HDFS-10731) | FSDirectory#verifyMaxDirItems does not log path name | Minor | namenode | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [YARN-5476](https://issues.apache.org/jira/browse/YARN-5476) | Not existed application reported as ACCEPTED state by YarnClientImpl | Critical | yarn | Yesha Vora | Junping Du | +| [YARN-5491](https://issues.apache.org/jira/browse/YARN-5491) | Random Failure TestCapacityScheduler#testCSQueueBlocked | Major | test | Bibin A Chundatt | Bibin A Chundatt | +| [HADOOP-13333](https://issues.apache.org/jira/browse/HADOOP-13333) | testConf.xml ls comparators in wrong order | Trivial | fs | John Zhuge | Vrushali C | +| [HADOOP-13470](https://issues.apache.org/jira/browse/HADOOP-13470) | GenericTestUtils$LogCapturer is flaky | Major | test, util | Mingliang Liu | Mingliang Liu | +| [HADOOP-13494](https://issues.apache.org/jira/browse/HADOOP-13494) | ReconfigurableBase can log sensitive information | Major | security | Sean Mackrory | Sean Mackrory | +| [HDFS-9530](https://issues.apache.org/jira/browse/HDFS-9530) | ReservedSpace is not cleared for abandoned Blocks | Critical | datanode | Fei Hui | Brahma Reddy Battula | +| [HDFS-10549](https://issues.apache.org/jira/browse/HDFS-10549) | Correctly revoke file leases when closing files | Major | hdfs-client | Yiqun Lin | Yiqun Lin | +| [HADOOP-13513](https://issues.apache.org/jira/browse/HADOOP-13513) | Java 1.7 support for org.apache.hadoop.fs.azure testcases | Minor | fs/azure | Tibor Kiss | Tibor Kiss | +| [HADOOP-13512](https://issues.apache.org/jira/browse/HADOOP-13512) | ReloadingX509TrustManager should keep reloading in case of exception | Critical | security | Mingliang Liu | Mingliang Liu | +| [YARN-4307](https://issues.apache.org/jira/browse/YARN-4307) | Display blacklisted nodes for AM container in the RM web UI | Major | resourcemanager, webapp | Naganarasimha G R | Naganarasimha G R | +| [MAPREDUCE-6763](https://issues.apache.org/jira/browse/MAPREDUCE-6763) | Shuffle server listen queue is too small | Major | mrv2 | Jason Lowe | Jason Lowe | +| [YARN-4837](https://issues.apache.org/jira/browse/YARN-4837) | User facing aspects of 'AM blacklisting' feature need fixing | Critical | . | Vinod Kumar Vavilapalli | Vinod Kumar Vavilapalli | +| [MAPREDUCE-6310](https://issues.apache.org/jira/browse/MAPREDUCE-6310) | Add jdiff support to MapReduce | Blocker | . | Li Lu | Li Lu | +| [YARN-3388](https://issues.apache.org/jira/browse/YARN-3388) | Allocation in LeafQueue could get stuck because DRF calculator isn't well supported when computing user-limit | Major | capacityscheduler | Nathan Roberts | Nathan Roberts | +| [YARN-4685](https://issues.apache.org/jira/browse/YARN-4685) | Disable AM blacklisting by default to mitigate situations that application get hanged | Critical | resourcemanager | Rohith Sharma K S | Rohith Sharma K S | +| [HADOOP-13428](https://issues.apache.org/jira/browse/HADOOP-13428) | Fix hadoop-common to generate jdiff | Blocker | . | Wangda Tan | Wangda Tan | +| [HDFS-10764](https://issues.apache.org/jira/browse/HDFS-10764) | Fix INodeFile#getBlocks to not return null | Major | namenode | Arpit Agarwal | Arpit Agarwal | +| [HDFS-10692](https://issues.apache.org/jira/browse/HDFS-10692) | Point JDiff base version for HDFS from 2.6.0 to 2.7.2 | Blocker | . | Wangda Tan | Wangda Tan | +| [HADOOP-13487](https://issues.apache.org/jira/browse/HADOOP-13487) | Hadoop KMS should load old delegation tokens from Zookeeper on startup | Major | kms | Alex Ivanov | Xiao Chen | +| [HDFS-10783](https://issues.apache.org/jira/browse/HDFS-10783) | The option '-maxSize' and '-step' fail in OfflineImageViewer | Major | tools | Yiqun Lin | Yiqun Lin | +| [HADOOP-13524](https://issues.apache.org/jira/browse/HADOOP-13524) | mvn eclipse:eclipse generates .gitignore'able files | Major | . | Vinod Kumar Vavilapalli | Vinod Kumar Vavilapalli | +| [HADOOP-13497](https://issues.apache.org/jira/browse/HADOOP-13497) | fix wrong command in CredentialProviderAPI.md | Trivial | documentation | Yuanbo Liu | Yuanbo Liu | +| [MAPREDUCE-6761](https://issues.apache.org/jira/browse/MAPREDUCE-6761) | Regression when handling providers - invalid configuration ServiceConfiguration causes Cluster initialization failure | Major | mrv2 | Peter Vary | Peter Vary | +| [HDFS-10748](https://issues.apache.org/jira/browse/HDFS-10748) | TestFileTruncate#testTruncateWithDataNodesRestart runs sometimes timeout | Major | test | Xiaoyu Yao | Yiqun Lin | +| [HDFS-10793](https://issues.apache.org/jira/browse/HDFS-10793) | Fix HdfsAuditLogger binary incompatibility introduced by HDFS-9184 | Blocker | . | Andrew Wang | Manoj Govindassamy | +| [HDFS-10652](https://issues.apache.org/jira/browse/HDFS-10652) | Add a unit test for HDFS-4660 | Major | datanode, hdfs | Yongjun Zhang | Vinayakumar B | +| [HADOOP-13552](https://issues.apache.org/jira/browse/HADOOP-13552) | RetryInvocationHandler logs all remote exceptions | Blocker | ipc | Jason Lowe | Jason Lowe | +| [HADOOP-12765](https://issues.apache.org/jira/browse/HADOOP-12765) | HttpServer2 should switch to using the non-blocking SslSelectChannelConnector to prevent performance degradation when handling SSL connections | Major | . | Min Shen | Min Shen | +| [MAPREDUCE-6768](https://issues.apache.org/jira/browse/MAPREDUCE-6768) | TestRecovery.testSpeculative failed with NPE | Major | mrv2 | Haibo Chen | Haibo Chen | +| [HADOOP-13559](https://issues.apache.org/jira/browse/HADOOP-13559) | Remove close() within try-with-resources in ChecksumFileSystem/ChecksumFs classes | Minor | fs | Aaron Fabbri | Aaron Fabbri | +| [HDFS-4210](https://issues.apache.org/jira/browse/HDFS-4210) | Throw helpful exception when DNS entry for JournalNode cannot be resolved | Trivial | ha, journal-node, namenode | Damien Hardy | John Zhuge | +| [MAPREDUCE-4784](https://issues.apache.org/jira/browse/MAPREDUCE-4784) | TestRecovery occasionally fails | Major | mrv2, test | Jason Lowe | Haibo Chen | +| [HDFS-10760](https://issues.apache.org/jira/browse/HDFS-10760) | DataXceiver#run() should not log InvalidToken exception as an error | Major | . | Pan Yuxuan | Pan Yuxuan | +| [HDFS-10729](https://issues.apache.org/jira/browse/HDFS-10729) | Improve log message for edit loading failures caused by FS limit checks. | Major | namenode | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [YARN-5221](https://issues.apache.org/jira/browse/YARN-5221) | Expose UpdateResourceRequest API to allow AM to request for change in container properties | Major | . | Arun Suresh | Arun Suresh | +| [HADOOP-13375](https://issues.apache.org/jira/browse/HADOOP-13375) | o.a.h.security.TestGroupsCaching.testBackgroundRefreshCounters seems flaky | Major | security, test | Mingliang Liu | Weiwei Yang | +| [YARN-5555](https://issues.apache.org/jira/browse/YARN-5555) | Scheduler UI: "% of Queue" is inaccurate if leaf queue is hierarchically nested. | Minor | . | Eric Payne | Eric Payne | +| [YARN-5549](https://issues.apache.org/jira/browse/YARN-5549) | AMLauncher#createAMContainerLaunchContext() should not log the command to be launched indiscriminately | Critical | resourcemanager | Daniel Templeton | Daniel Templeton | +| [HDFS-10841](https://issues.apache.org/jira/browse/HDFS-10841) | Remove duplicate or unused variable in appendFile() | Minor | . | Kihwal Lee | Kihwal Lee | +| [HADOOP-13558](https://issues.apache.org/jira/browse/HADOOP-13558) | UserGroupInformation created from a Subject incorrectly tries to renew the Kerberos ticket | Major | security | Alejandro Abdelnur | Xiao Chen | +| [HDFS-9038](https://issues.apache.org/jira/browse/HDFS-9038) | DFS reserved space is erroneously counted towards non-DFS used. | Major | datanode | Chris Nauroth | Brahma Reddy Battula | +| [HDFS-10832](https://issues.apache.org/jira/browse/HDFS-10832) | Propagate ACL bit and isEncrypted bit in HttpFS FileStatus permissions | Critical | httpfs | Andrew Wang | Andrew Wang | +| [YARN-5632](https://issues.apache.org/jira/browse/YARN-5632) | UPDATE\_EXECUTION\_TYPE causes UpdateContainerRequestPBImpl to throw | Major | api | Jason Lowe | Jason Lowe | +| [HDFS-9781](https://issues.apache.org/jira/browse/HDFS-9781) | FsDatasetImpl#getBlockReports can occasionally throw NullPointerException | Major | datanode | Wei-Chiu Chuang | Manoj Govindassamy | +| [HDFS-10830](https://issues.apache.org/jira/browse/HDFS-10830) | FsDatasetImpl#removeVolumes crashes with IllegalMonitorStateException when vol being removed is in use | Major | hdfs | Manoj Govindassamy | Arpit Agarwal | +| [YARN-5190](https://issues.apache.org/jira/browse/YARN-5190) | Registering/unregistering container metrics triggered by ContainerEvent and ContainersMonitorEvent are conflict which cause uncaught exception in ContainerMonitorImpl | Blocker | . | Junping Du | Junping Du | +| [HDFS-10856](https://issues.apache.org/jira/browse/HDFS-10856) | Update the comment of BPServiceActor$Scheduler#scheduleNextBlockReport | Minor | documentation | Akira Ajisaka | Yiqun Lin | +| [YARN-5630](https://issues.apache.org/jira/browse/YARN-5630) | NM fails to start after downgrade from 2.8 to 2.7 | Blocker | nodemanager | Jason Lowe | Jason Lowe | +| [HADOOP-13616](https://issues.apache.org/jira/browse/HADOOP-13616) | Broken code snippet area in Hadoop Benchmarking | Minor | documentation | Kai Sasaki | Kai Sasaki | +| [HDFS-10862](https://issues.apache.org/jira/browse/HDFS-10862) | Typos in 4 log messages | Trivial | . | Mehran Hassani | Mehran Hassani | +| [YARN-4232](https://issues.apache.org/jira/browse/YARN-4232) | TopCLI console support for HA mode | Minor | . | Bibin A Chundatt | Bibin A Chundatt | +| [YARN-5655](https://issues.apache.org/jira/browse/YARN-5655) | TestContainerManagerSecurity#testNMTokens is asserting | Major | . | Jason Lowe | Robert Kanter | +| [HDFS-10879](https://issues.apache.org/jira/browse/HDFS-10879) | TestEncryptionZonesWithKMS#testReadWrite fails intermittently | Major | . | Xiao Chen | Xiao Chen | +| [YARN-5539](https://issues.apache.org/jira/browse/YARN-5539) | TimelineClient failed to retry on "java.net.SocketTimeoutException: Read timed out" | Critical | yarn | Sumana Sathish | Junping Du | +| [HADOOP-13643](https://issues.apache.org/jira/browse/HADOOP-13643) | Math error in AbstractContractDistCpTest | Minor | . | Aaron Fabbri | Aaron Fabbri | +| [HDFS-10886](https://issues.apache.org/jira/browse/HDFS-10886) | Replace "fs.default.name" with "fs.defaultFS" in viewfs document | Minor | documentation, federation | Brahma Reddy Battula | Brahma Reddy Battula | +| [HADOOP-13535](https://issues.apache.org/jira/browse/HADOOP-13535) | Add jetty6 acceptor startup issue workaround to branch-2 | Major | . | Wei-Chiu Chuang | Min Shen | +| [YARN-5664](https://issues.apache.org/jira/browse/YARN-5664) | Fix Yarn documentation to link to correct versions. | Minor | . | Xiao Chen | Xiao Chen | +| [YARN-5663](https://issues.apache.org/jira/browse/YARN-5663) | Small refactor in ZKRMStateStore | Minor | resourcemanager | Oleksii Dymytrov | Oleksii Dymytrov | +| [HADOOP-12597](https://issues.apache.org/jira/browse/HADOOP-12597) | In kms-site.xml configuration "hadoop.security.keystore.JavaKeyStoreProvider.password" should be updated with new name | Minor | security | huangyitian | Surendra Singh Lilhore | +| [HADOOP-13638](https://issues.apache.org/jira/browse/HADOOP-13638) | KMS should set UGI's Configuration object properly | Major | kms | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HDFS-9885](https://issues.apache.org/jira/browse/HDFS-9885) | Correct the distcp counters name while displaying counters | Minor | distcp | Archana T | Surendra Singh Lilhore | +| [YARN-5660](https://issues.apache.org/jira/browse/YARN-5660) | Wrong audit constants are used in Get/Put of priority in RMWebService | Trivial | resourcemanager | Rohith Sharma K S | Rohith Sharma K S | +| [HDFS-10889](https://issues.apache.org/jira/browse/HDFS-10889) | Remove outdated Fault Injection Framework documentaion | Major | documentation | Brahma Reddy Battula | Brahma Reddy Battula | +| [HDFS-10713](https://issues.apache.org/jira/browse/HDFS-10713) | Throttle FsNameSystem lock warnings | Major | logging, namenode | Arpit Agarwal | Hanisha Koneru | +| [HDFS-10426](https://issues.apache.org/jira/browse/HDFS-10426) | TestPendingInvalidateBlock failed in trunk | Major | test | Yiqun Lin | Yiqun Lin | +| [HDFS-10828](https://issues.apache.org/jira/browse/HDFS-10828) | Fix usage of FsDatasetImpl object lock in ReplicaMap | Blocker | . | Arpit Agarwal | Arpit Agarwal | +| [YARN-5631](https://issues.apache.org/jira/browse/YARN-5631) | Missing refreshClusterMaxPriority usage in rmadmin help message | Minor | . | Kai Sasaki | Kai Sasaki | +| [HDFS-9444](https://issues.apache.org/jira/browse/HDFS-9444) | Add utility to find set of available ephemeral ports to ServerSocketUtil | Major | . | Brahma Reddy Battula | Masatake Iwasaki | +| [HDFS-10824](https://issues.apache.org/jira/browse/HDFS-10824) | MiniDFSCluster#storageCapacities has no effects on real capacity | Major | . | Xiaobing Zhou | Xiaobing Zhou | +| [HDFS-10914](https://issues.apache.org/jira/browse/HDFS-10914) | Move remnants of oah.hdfs.client to hadoop-hdfs-client | Critical | hdfs-client | Andrew Wang | Andrew Wang | +| [MAPREDUCE-6771](https://issues.apache.org/jira/browse/MAPREDUCE-6771) | RMContainerAllocator sends container diagnostics event after corresponding completion event | Major | mrv2 | Haibo Chen | Haibo Chen | +| [HADOOP-13640](https://issues.apache.org/jira/browse/HADOOP-13640) | Fix findbugs warning in VersionInfoMojo.java | Major | . | Tsuyoshi Ozawa | Yuanbo Liu | +| [HDFS-10850](https://issues.apache.org/jira/browse/HDFS-10850) | getEZForPath should NOT throw FNF | Blocker | hdfs | Daryn Sharp | Andrew Wang | +| [HDFS-10945](https://issues.apache.org/jira/browse/HDFS-10945) | Fix the Findbugwaring FSNamesystem#renameTo(String, String, boolean) in branch-2 | Major | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [HDFS-10810](https://issues.apache.org/jira/browse/HDFS-10810) | Setreplication removing block from underconstrcution temporarily when batch IBR is enabled. | Major | namenode | Brahma Reddy Battula | Brahma Reddy Battula | +| [HDFS-10944](https://issues.apache.org/jira/browse/HDFS-10944) | Correct the javadoc of dfsadmin#disallowSnapshot | Minor | documentation | Jagadesh Kiran N | Jagadesh Kiran N | +| [HDFS-10947](https://issues.apache.org/jira/browse/HDFS-10947) | Correct the API name for truncate in webhdfs document | Major | documentation | Surendra Singh Lilhore | Surendra Singh Lilhore | +| [HADOOP-12667](https://issues.apache.org/jira/browse/HADOOP-12667) | s3a: Support createNonRecursive API | Major | fs/s3 | Sean Mackrory | Sean Mackrory | +| [HDFS-10609](https://issues.apache.org/jira/browse/HDFS-10609) | Uncaught InvalidEncryptionKeyException during pipeline recovery may abort downstream applications | Major | encryption | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HDFS-10962](https://issues.apache.org/jira/browse/HDFS-10962) | TestRequestHedgingProxyProvider is flaky | Major | test | Andrew Wang | Andrew Wang | +| [MAPREDUCE-6740](https://issues.apache.org/jira/browse/MAPREDUCE-6740) | Enforce mapreduce.task.timeout to be at least mapreduce.task.progress-report.interval | Minor | mr-am | Haibo Chen | Haibo Chen | +| [YARN-5101](https://issues.apache.org/jira/browse/YARN-5101) | YARN\_APPLICATION\_UPDATED event is parsed in ApplicationHistoryManagerOnTimelineStore#convertToApplicationReport with reversed order | Major | . | Xuan Gong | Sunil G | +| [YARN-5659](https://issues.apache.org/jira/browse/YARN-5659) | getPathFromYarnURL should use standard methods | Major | . | Sergey Shelukhin | Sergey Shelukhin | +| [HADOOP-12611](https://issues.apache.org/jira/browse/HADOOP-12611) | TestZKSignerSecretProvider#testMultipleInit occasionally fail | Major | . | Wei-Chiu Chuang | Eric Badger | +| [HDFS-10797](https://issues.apache.org/jira/browse/HDFS-10797) | Disk usage summary of snapshots causes renamed blocks to get counted twice | Major | snapshots | Sean Mackrory | Sean Mackrory | +| [HDFS-10991](https://issues.apache.org/jira/browse/HDFS-10991) | Export hdfsTruncateFile symbol in libhdfs | Blocker | libhdfs | Surendra Singh Lilhore | Surendra Singh Lilhore | +| [HADOOP-13700](https://issues.apache.org/jira/browse/HADOOP-13700) | Remove unthrown IOException from TrashPolicy#initialize and #getInstance signatures | Critical | fs | Haibo Chen | Andrew Wang | +| [HDFS-11002](https://issues.apache.org/jira/browse/HDFS-11002) | Fix broken attr/getfattr/setfattr links in ExtendedAttributes.md | Major | documentation | Mingliang Liu | Mingliang Liu | +| [HDFS-11000](https://issues.apache.org/jira/browse/HDFS-11000) | webhdfs PUT does not work if requests are routed to call queue. | Major | . | Kihwal Lee | Kihwal Lee | +| [HDFS-10987](https://issues.apache.org/jira/browse/HDFS-10987) | Make Decommission less expensive when lot of blocks present. | Critical | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [HADOOP-13024](https://issues.apache.org/jira/browse/HADOOP-13024) | Distcp with -delete feature on raw data not implemented | Major | tools/distcp | Mavin Martin | Mavin Martin | +| [HDFS-10986](https://issues.apache.org/jira/browse/HDFS-10986) | DFSAdmin should log detailed error message if any | Major | tools | Mingliang Liu | Mingliang Liu | +| [HDFS-10990](https://issues.apache.org/jira/browse/HDFS-10990) | TestPendingInvalidateBlock should wait for IBRs | Minor | . | Yiqun Lin | Yiqun Lin | +| [HDFS-10735](https://issues.apache.org/jira/browse/HDFS-10735) | Distcp using webhdfs on secure HA clusters fails with StandbyException | Major | webhdfs | Benoy Antony | Benoy Antony | +| [HDFS-10883](https://issues.apache.org/jira/browse/HDFS-10883) | `getTrashRoot`'s behavior is not consistent in DFS after enabling EZ. | Major | . | Yuanbo Liu | Yuanbo Liu | +| [HADOOP-13707](https://issues.apache.org/jira/browse/HADOOP-13707) | If kerberos is enabled while HTTP SPNEGO is not configured, some links cannot be accessed | Major | . | Yuanbo Liu | Yuanbo Liu | +| [HDFS-10301](https://issues.apache.org/jira/browse/HDFS-10301) | BlockReport retransmissions may lead to storages falsely being declared zombie if storage report processing happens out of order | Critical | namenode | Konstantin Shvachko | Vinitha Reddy Gankidi | +| [HDFS-10712](https://issues.apache.org/jira/browse/HDFS-10712) | Fix TestDataNodeVolumeFailure on 2.\* branches. | Major | . | Konstantin Shvachko | Vinitha Reddy Gankidi | +| [HDFS-10920](https://issues.apache.org/jira/browse/HDFS-10920) | TestStorageMover#testNoSpaceDisk is failing intermittently | Major | test | Rakesh R | Rakesh R | +| [HDFS-10960](https://issues.apache.org/jira/browse/HDFS-10960) | TestDataNodeHotSwapVolumes#testRemoveVolumeBeingWritten fails at disk error verification after volume remove | Minor | hdfs | Manoj Govindassamy | Manoj Govindassamy | +| [HDFS-10752](https://issues.apache.org/jira/browse/HDFS-10752) | Several log refactoring/improvement suggestion in HDFS | Major | . | Nemo Chen | Hanisha Koneru | +| [HDFS-11025](https://issues.apache.org/jira/browse/HDFS-11025) | TestDiskspaceQuotaUpdate fails in trunk due to Bind exception | Minor | . | Yiqun Lin | Yiqun Lin | +| [HDFS-11018](https://issues.apache.org/jira/browse/HDFS-11018) | Incorrect check and message in FsDatasetImpl#invalidate | Major | datanode | Wei-Chiu Chuang | Yiqun Lin | +| [HADOOP-13236](https://issues.apache.org/jira/browse/HADOOP-13236) | truncate will fail when we use viewfilesystem | Major | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [HADOOP-13749](https://issues.apache.org/jira/browse/HADOOP-13749) | KMSClientProvider combined with KeyProviderCache can result in wrong UGI being used | Critical | . | Sergey Shelukhin | Xiaoyu Yao | +| [HDFS-11042](https://issues.apache.org/jira/browse/HDFS-11042) | Add missing cleanupSSLConfig() call for tests that use setupSSLConfig() | Major | test | Kuhu Shukla | Kuhu Shukla | +| [HDFS-11015](https://issues.apache.org/jira/browse/HDFS-11015) | Enforce timeout in balancer | Major | balancer & mover | Kihwal Lee | Kihwal Lee | +| [YARN-5677](https://issues.apache.org/jira/browse/YARN-5677) | RM should transition to standby when connection is lost for an extended period | Critical | resourcemanager | Daniel Templeton | Daniel Templeton | +| [HDFS-11054](https://issues.apache.org/jira/browse/HDFS-11054) | Suppress verbose log message in BlockPlacementPolicyDefault | Major | . | Arpit Agarwal | Chen Liang | +| [HDFS-11050](https://issues.apache.org/jira/browse/HDFS-11050) | Change log level to 'warn' when ssl initialization fails and defaults to DEFAULT\_TIMEOUT\_CONN\_CONFIGURATOR | Major | . | Kuhu Shukla | Kuhu Shukla | +| [HDFS-8492](https://issues.apache.org/jira/browse/HDFS-8492) | DN should notify NN when client requests a missing block | Major | . | Daryn Sharp | Walter Su | +| [MAPREDUCE-6541](https://issues.apache.org/jira/browse/MAPREDUCE-6541) | Exclude scheduled reducer memory when calculating available mapper slots from headroom to avoid deadlock | Major | . | Wangda Tan | Varun Saxena | +| [YARN-3848](https://issues.apache.org/jira/browse/YARN-3848) | TestNodeLabelContainerAllocation is not timing out | Major | test | Jason Lowe | Varun Saxena | +| [HADOOP-13201](https://issues.apache.org/jira/browse/HADOOP-13201) | Print the directory paths when ViewFs denies the rename operation on internal dirs | Major | viewfs | Tianyin Xu | Rakesh R | +| [YARN-4831](https://issues.apache.org/jira/browse/YARN-4831) | Recovered containers will be killed after NM stateful restart | Major | nodemanager | Siqi Li | Siqi Li | +| [YARN-3432](https://issues.apache.org/jira/browse/YARN-3432) | Cluster metrics have wrong Total Memory when there is reserved memory on CS | Major | capacityscheduler, resourcemanager | Thomas Graves | Brahma Reddy Battula | +| [HDFS-9500](https://issues.apache.org/jira/browse/HDFS-9500) | datanodesSoftwareVersions map may counting wrong when rolling upgrade | Major | . | Phil Yang | Erik Krogen | +| [MAPREDUCE-2631](https://issues.apache.org/jira/browse/MAPREDUCE-2631) | Potential resource leaks in BinaryProtocol$TeeOutputStream.java | Major | . | Ravi Teja Ch N V | Sunil G | +| [HADOOP-13770](https://issues.apache.org/jira/browse/HADOOP-13770) | Shell.checkIsBashSupported swallowed an interrupted exception | Blocker | util | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [YARN-5027](https://issues.apache.org/jira/browse/YARN-5027) | NM should clean up app log dirs after NM restart | Major | nodemanager | sandflee | sandflee | +| [YARN-5767](https://issues.apache.org/jira/browse/YARN-5767) | Fix the order that resources are cleaned up from the local Public/Private caches | Major | . | Chris Trezzo | Chris Trezzo | +| [HDFS-11061](https://issues.apache.org/jira/browse/HDFS-11061) | Update dfs -count -t command line help and documentation | Minor | documentation, fs | Wei-Chiu Chuang | Yiqun Lin | +| [YARN-5773](https://issues.apache.org/jira/browse/YARN-5773) | RM recovery too slow due to LeafQueue#activateApplication() | Critical | capacity scheduler, rolling upgrade | Bibin A Chundatt | Bibin A Chundatt | +| [HDFS-10455](https://issues.apache.org/jira/browse/HDFS-10455) | Logging the username when deny the setOwner operation | Minor | namenode | Tianyin Xu | Rakesh R | +| [HADOOP-13773](https://issues.apache.org/jira/browse/HADOOP-13773) | wrong HADOOP\_CLIENT\_OPTS in hadoop-env on branch-2 | Major | conf | Fei Hui | Fei Hui | +| [YARN-5001](https://issues.apache.org/jira/browse/YARN-5001) | Aggregated Logs root directory is created with wrong group if nonexistent | Major | log-aggregation, nodemanager, security | Haibo Chen | Haibo Chen | +| [YARN-5815](https://issues.apache.org/jira/browse/YARN-5815) | Random failure of TestApplicationPriority.testOrderOfActivatingThePriorityApplicationOnRMRestart | Major | . | Bibin A Chundatt | Bibin A Chundatt | +| [HDFS-11097](https://issues.apache.org/jira/browse/HDFS-11097) | Fix the jenkins warning related to the deprecated method StorageReceivedDeletedBlocks | Major | . | Yiqun Lin | Yiqun Lin | +| [YARN-5837](https://issues.apache.org/jira/browse/YARN-5837) | NPE when getting node status of a decommissioned node after an RM restart | Major | . | Robert Kanter | Robert Kanter | +| [HADOOP-13798](https://issues.apache.org/jira/browse/HADOOP-13798) | TestHadoopArchives times out | Major | test | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-13797](https://issues.apache.org/jira/browse/HADOOP-13797) | Remove hardcoded absolute path for ls | Major | util | Christine Koppelt | Christine Koppelt | +| [HADOOP-13804](https://issues.apache.org/jira/browse/HADOOP-13804) | MutableStat mean loses accuracy if add(long, long) is used | Minor | metrics | Erik Krogen | Erik Krogen | +| [HDFS-11128](https://issues.apache.org/jira/browse/HDFS-11128) | CreateEditsLog throws NullPointerException | Major | hdfs | Hanisha Koneru | Hanisha Koneru | +| [HDFS-11129](https://issues.apache.org/jira/browse/HDFS-11129) | TestAppendSnapshotTruncate fails with bind exception | Major | test | Brahma Reddy Battula | Brahma Reddy Battula | +| [HADOOP-13813](https://issues.apache.org/jira/browse/HADOOP-13813) | TestDelegationTokenFetcher#testDelegationTokenWithoutRenewer is failing | Major | security, test | Mingliang Liu | Mingliang Liu | +| [HDFS-11087](https://issues.apache.org/jira/browse/HDFS-11087) | NamenodeFsck should check if the output writer is still writable. | Major | namenode | Konstantin Shvachko | Erik Krogen | +| [HDFS-11135](https://issues.apache.org/jira/browse/HDFS-11135) | The tests in TestBalancer run fails due to NPE | Major | test | Yiqun Lin | Yiqun Lin | +| [HDFS-11056](https://issues.apache.org/jira/browse/HDFS-11056) | Concurrent append and read operations lead to checksum error | Major | datanode, httpfs | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [MAPREDUCE-6797](https://issues.apache.org/jira/browse/MAPREDUCE-6797) | Job history server scans can become blocked on a single, slow entry | Critical | jobhistoryserver | Prabhu Joseph | Prabhu Joseph | +| [YARN-4355](https://issues.apache.org/jira/browse/YARN-4355) | NPE while processing localizer heartbeat | Major | nodemanager | Jason Lowe | Varun Saxena | +| [YARN-4218](https://issues.apache.org/jira/browse/YARN-4218) | Metric for resource\*time that was preempted | Major | resourcemanager | Chang Li | Chang Li | +| [YARN-5875](https://issues.apache.org/jira/browse/YARN-5875) | TestTokenClientRMService#testTokenRenewalWrongUser fails | Major | . | Varun Saxena | Gergely Novák | +| [HADOOP-13815](https://issues.apache.org/jira/browse/HADOOP-13815) | TestKMS#testDelegationTokensOpsSimple and TestKMS#testDelegationTokensOpsKerberized Fails in Trunk | Major | test | Brahma Reddy Battula | Xiao Chen | +| [YARN-5836](https://issues.apache.org/jira/browse/YARN-5836) | Malicious AM can kill containers of other apps running in any node its containers are running | Minor | nodemanager | Botong Huang | Botong Huang | +| [HDFS-11134](https://issues.apache.org/jira/browse/HDFS-11134) | Fix bind exception threw in TestRenameWhileOpen | Major | . | Yiqun Lin | Yiqun Lin | +| [MAPREDUCE-6801](https://issues.apache.org/jira/browse/MAPREDUCE-6801) | Fix flaky TestKill.testKillJob() | Major | mrv2 | Haibo Chen | Haibo Chen | +| [HADOOP-13814](https://issues.apache.org/jira/browse/HADOOP-13814) | Sample configuration of KMS HTTP Authentication signature is misleading | Minor | conf, documentation, kms | Masahiro Tanaka | Masahiro Tanaka | +| [HDFS-11144](https://issues.apache.org/jira/browse/HDFS-11144) | TestFileCreationDelete#testFileCreationDeleteParent fails wind bind exception | Major | test | Brahma Reddy Battula | Brahma Reddy Battula | +| [HDFS-11101](https://issues.apache.org/jira/browse/HDFS-11101) | TestDFSShell#testMoveWithTargetPortEmpty fails intermittently | Major | test | Brahma Reddy Battula | Brahma Reddy Battula | +| [YARN-5859](https://issues.apache.org/jira/browse/YARN-5859) | TestResourceLocalizationService#testParallelDownloadAttemptsForPublicResource sometimes fails | Major | test | Jason Lowe | Eric Badger | +| [HADOOP-13663](https://issues.apache.org/jira/browse/HADOOP-13663) | Index out of range in SysInfoWindows | Major | scripts | Inigo Goiri | Inigo Goiri | +| [HDFS-11174](https://issues.apache.org/jira/browse/HDFS-11174) | Wrong HttpFS test command in doc | Minor | documentation, httpfs | John Zhuge | John Zhuge | +| [HADOOP-13820](https://issues.apache.org/jira/browse/HADOOP-13820) | Replace ugi.getUsername() with ugi.getShortUserName() in viewFS | Minor | viewfs | Archana T | Brahma Reddy Battula | +| [HADOOP-13838](https://issues.apache.org/jira/browse/HADOOP-13838) | KMSTokenRenewer should close providers | Critical | kms | Xiao Chen | Xiao Chen | +| [HADOOP-13830](https://issues.apache.org/jira/browse/HADOOP-13830) | Intermittent failure of ITestS3NContractRootDir#testRecursiveRootListing: "Can not create a Path from an empty string" | Minor | fs/s3, test | Steve Loughran | Steve Loughran | +| [YARN-5915](https://issues.apache.org/jira/browse/YARN-5915) | ATS 1.5 FileSystemTimelineWriter causes flush() to be called after every event write | Major | timelineserver | Atul Sikaria | Atul Sikaria | +| [MAPREDUCE-6815](https://issues.apache.org/jira/browse/MAPREDUCE-6815) | Fix flaky TestKill.testKillTask() | Major | mrv2 | Haibo Chen | Haibo Chen | +| [HDFS-11181](https://issues.apache.org/jira/browse/HDFS-11181) | Fuse wrapper has a typo | Trivial | fuse-dfs | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HADOOP-13847](https://issues.apache.org/jira/browse/HADOOP-13847) | KMSWebApp should close KeyProviderCryptoExtension | Major | kms | Anthony Young-Garner | John Zhuge | +| [YARN-5559](https://issues.apache.org/jira/browse/YARN-5559) | Analyse 2.8.0/3.0.0 jdiff reports and fix any issues | Blocker | resourcemanager | Wangda Tan | Akira Ajisaka | +| [HADOOP-13861](https://issues.apache.org/jira/browse/HADOOP-13861) | Spelling errors in logging and exceptions for code | Major | common, fs, io, security | Grant Sohn | Grant Sohn | +| [HDFS-11180](https://issues.apache.org/jira/browse/HDFS-11180) | Intermittent deadlock in NameNode when failover happens. | Blocker | namenode | Abhishek Modi | Akira Ajisaka | +| [HDFS-11198](https://issues.apache.org/jira/browse/HDFS-11198) | NN UI should link DN web address using hostnames | Critical | . | Kihwal Lee | Weiwei Yang | +| [MAPREDUCE-6816](https://issues.apache.org/jira/browse/MAPREDUCE-6816) | Progress bars in Web UI always at 100% | Blocker | webapps | Shen Yinjie | Shen Yinjie | +| [YARN-5184](https://issues.apache.org/jira/browse/YARN-5184) | Fix up incompatible changes introduced on ContainerStatus and NodeReport | Blocker | api | Karthik Kambatla | Sangjin Lee | +| [YARN-5921](https://issues.apache.org/jira/browse/YARN-5921) | Incorrect synchronization in RMContextImpl#setHAServiceState/getHAServiceState | Major | . | Varun Saxena | Varun Saxena | +| [HDFS-11140](https://issues.apache.org/jira/browse/HDFS-11140) | Directory Scanner should log startup message time correctly | Minor | . | Xiao Chen | Yiqun Lin | +| [HDFS-11223](https://issues.apache.org/jira/browse/HDFS-11223) | Fix typos in HttpFs documentations | Trivial | documentation | Yiqun Lin | Yiqun Lin | +| [HDFS-11197](https://issues.apache.org/jira/browse/HDFS-11197) | Listing encryption zones fails when deleting a EZ that is on a snapshotted directory | Minor | hdfs | Wellington Chevreuil | Wellington Chevreuil | +| [HDFS-11224](https://issues.apache.org/jira/browse/HDFS-11224) | Lifeline message should be ignored for dead nodes | Critical | . | Vinayakumar B | Vinayakumar B | +| [HADOOP-13824](https://issues.apache.org/jira/browse/HADOOP-13824) | FsShell can suppress the real error if no error message is present | Major | fs | Rob Vesse | John Zhuge | +| [MAPREDUCE-6820](https://issues.apache.org/jira/browse/MAPREDUCE-6820) | Fix dead links in Job relevant classes | Minor | documentation | Yiqun Lin | Yiqun Lin | +| [HDFS-8870](https://issues.apache.org/jira/browse/HDFS-8870) | Lease is leaked on write failure | Major | hdfs-client | Rushabh S Shah | Kuhu Shukla | +| [HADOOP-13565](https://issues.apache.org/jira/browse/HADOOP-13565) | KerberosAuthenticationHandler#authenticate should not rebuild SPN based on client request | Major | security | Xiaoyu Yao | Xiaoyu Yao | +| [HDFS-11229](https://issues.apache.org/jira/browse/HDFS-11229) | HDFS-11056 failed to close meta file | Blocker | datanode | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HDFS-10570](https://issues.apache.org/jira/browse/HDFS-10570) | Remove classpath conflicts of netty-all jar in hadoop-hdfs-client | Minor | test | Vinayakumar B | Vinayakumar B | +| [HDFS-10684](https://issues.apache.org/jira/browse/HDFS-10684) | WebHDFS DataNode calls fail without parameter createparent | Blocker | webhdfs | Samuel Low | John Zhuge | +| [HDFS-11204](https://issues.apache.org/jira/browse/HDFS-11204) | Document the missing options of hdfs zkfc command | Minor | documentation | Yiqun Lin | Yiqun Lin | +| [HADOOP-13890](https://issues.apache.org/jira/browse/HADOOP-13890) | Maintain HTTP/host as SPNEGO SPN support and fix KerberosName parsing | Major | test | Brahma Reddy Battula | Xiaoyu Yao | +| [HDFS-11094](https://issues.apache.org/jira/browse/HDFS-11094) | Send back HAState along with NamespaceInfo during a versionRequest as an optional parameter | Major | datanode | Eric Badger | Eric Badger | +| [HDFS-11160](https://issues.apache.org/jira/browse/HDFS-11160) | VolumeScanner reports write-in-progress replicas as corrupt incorrectly | Major | datanode | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HDFS-11263](https://issues.apache.org/jira/browse/HDFS-11263) | ClassCastException when we use Bzipcodec for Fsimage compression | Critical | . | Brahma Reddy Battula | Brahma Reddy Battula | +| [HDFS-11195](https://issues.apache.org/jira/browse/HDFS-11195) | Return error when appending files by webhdfs rest api fails | Major | . | Yuanbo Liu | Yuanbo Liu | +| [HDFS-11261](https://issues.apache.org/jira/browse/HDFS-11261) | Document missing NameNode metrics | Minor | documentation | Yiqun Lin | Yiqun Lin | +| [HDFS-11258](https://issues.apache.org/jira/browse/HDFS-11258) | File mtime change could not save to editlog | Critical | . | Jimmy Xiang | Jimmy Xiang | +| [HDFS-11271](https://issues.apache.org/jira/browse/HDFS-11271) | Typo in NameNode UI | Trivial | namenode, ui | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HDFS-11250](https://issues.apache.org/jira/browse/HDFS-11250) | Fix a typo in ReplicaUnderRecovery#setRecoveryID | Trivial | . | Yiqun Lin | Yiqun Lin | +| [HDFS-11270](https://issues.apache.org/jira/browse/HDFS-11270) | Document the missing options of NameNode bootstrap command | Minor | documentation | Yiqun Lin | Yiqun Lin | +| [HDFS-11252](https://issues.apache.org/jira/browse/HDFS-11252) | TestFileTruncate#testTruncateWithDataNodesRestartImmediately can fail with BindException | Major | . | Jason Lowe | Yiqun Lin | +| [YARN-6024](https://issues.apache.org/jira/browse/YARN-6024) | Capacity Scheduler 'continuous reservation looking' doesn't work when sum of queue's used and reserved resources is equal to max | Major | . | Wangda Tan | Wangda Tan | +| [HADOOP-13883](https://issues.apache.org/jira/browse/HADOOP-13883) | Add description of -fs option in generic command usage | Minor | documentation | Yiqun Lin | Yiqun Lin | +| [YARN-6029](https://issues.apache.org/jira/browse/YARN-6029) | CapacityScheduler deadlock when ParentQueue#getQueueUserAclInfo is called by one thread and LeafQueue#assignContainers is releasing excessive reserved container by another thread | Critical | capacityscheduler | Tao Yang | Tao Yang | +| [HADOOP-12733](https://issues.apache.org/jira/browse/HADOOP-12733) | Remove references to obsolete io.seqfile configuration variables | Minor | . | Ray Chiang | Ray Chiang | +| [HDFS-11280](https://issues.apache.org/jira/browse/HDFS-11280) | Allow WebHDFS to reuse HTTP connections to NN | Major | hdfs | Zheng Shao | Zheng Shao | +| [MAPREDUCE-6711](https://issues.apache.org/jira/browse/MAPREDUCE-6711) | JobImpl fails to handle preemption events on state COMMITTING | Major | . | Li Lu | Prabhu Joseph | +| [HADOOP-13958](https://issues.apache.org/jira/browse/HADOOP-13958) | Bump up release year to 2017 | Blocker | . | Junping Du | Junping Du | +| [YARN-6068](https://issues.apache.org/jira/browse/YARN-6068) | Log aggregation get failed when NM restart even with recovery | Blocker | . | Junping Du | Junping Du | +| [YARN-4148](https://issues.apache.org/jira/browse/YARN-4148) | When killing app, RM releases app's resource before they are released by NM | Major | resourcemanager | Jun Gong | Jason Lowe | +| [HDFS-11312](https://issues.apache.org/jira/browse/HDFS-11312) | Fix incompatible tag number change for nonDfsUsed in DatanodeInfoProto | Blocker | . | Sean Mackrory | Sean Mackrory | +| [YARN-6072](https://issues.apache.org/jira/browse/YARN-6072) | RM unable to start in secure mode | Blocker | resourcemanager | Bibin A Chundatt | Ajith S | +| [HDFS-10733](https://issues.apache.org/jira/browse/HDFS-10733) | NameNode terminated after full GC thinking QJM is unresponsive. | Major | namenode, qjm | Konstantin Shvachko | Vinitha Reddy Gankidi | +| [HADOOP-14001](https://issues.apache.org/jira/browse/HADOOP-14001) | Improve delegation token validity checking | Major | . | Akira Ajisaka | Akira Ajisaka | +| [HDFS-11376](https://issues.apache.org/jira/browse/HDFS-11376) | Revert HDFS-8377 Support HTTP/2 in datanode | Major | datanode | Andrew Wang | Xiao Chen | +| [YARN-6151](https://issues.apache.org/jira/browse/YARN-6151) | FS preemption does not consider child queues over fairshare if the parent is under | Major | fairscheduler | Yufei Gu | Yufei Gu | +| [YARN-5271](https://issues.apache.org/jira/browse/YARN-5271) | ATS client doesn't work with Jersey 2 on the classpath | Major | client, timelineserver | Steve Loughran | Weiwei Yang | +| [YARN-3933](https://issues.apache.org/jira/browse/YARN-3933) | FairScheduler: Multiple calls to completedContainer are not safe | Major | fairscheduler | Lavkesh Lahngir | Shiwei Guo | +| [YARN-6177](https://issues.apache.org/jira/browse/YARN-6177) | Yarn client should exit with an informative error message if an incompatible Jersey library is used at client | Major | . | Weiwei Yang | Weiwei Yang | +| [HDFS-11379](https://issues.apache.org/jira/browse/HDFS-11379) | DFSInputStream may infinite loop requesting block locations | Critical | hdfs-client | Daryn Sharp | Daryn Sharp | +| [HADOOP-14092](https://issues.apache.org/jira/browse/HADOOP-14092) | Typo in hadoop-aws index.md | Trivial | fs/s3 | John Zhuge | John Zhuge | +| [HADOOP-13826](https://issues.apache.org/jira/browse/HADOOP-13826) | S3A Deadlock in multipart copy due to thread pool limits. | Critical | fs/s3 | Sean Mackrory | Sean Mackrory | +| [HADOOP-14017](https://issues.apache.org/jira/browse/HADOOP-14017) | User friendly name for ADLS user and group | Major | fs/adl | John Zhuge | Vishwajeet Dusane | +| [YARN-6175](https://issues.apache.org/jira/browse/YARN-6175) | FairScheduler: Negative vcore for resource needed to preempt | Major | fairscheduler | Yufei Gu | Yufei Gu | +| [HADOOP-14028](https://issues.apache.org/jira/browse/HADOOP-14028) | S3A BlockOutputStreams doesn't delete temporary files in multipart uploads or handle part upload failures | Critical | fs/s3 | Seth Fitzsimmons | Steve Loughran | +| [YARN-1728](https://issues.apache.org/jira/browse/YARN-1728) | Workaround guice3x-undecoded pathInfo in YARN WebApp | Major | . | Abraham Elmahrek | Yuanbo Liu | +| [HADOOP-12979](https://issues.apache.org/jira/browse/HADOOP-12979) | IOE in S3a: ${hadoop.tmp.dir}/s3a not configured | Major | . | Steve Loughran | Steve Loughran | +| [YARN-6270](https://issues.apache.org/jira/browse/YARN-6270) | WebUtils.getRMWebAppURLWithScheme() needs to honor RM HA setting | Major | . | Sumana Sathish | Xuan Gong | +| [HDFS-11498](https://issues.apache.org/jira/browse/HDFS-11498) | Make RestCsrfPreventionHandler and WebHdfsHandler compatible with Netty 4.0 | Major | . | Andrew Wang | Andrew Wang | +| [HADOOP-14087](https://issues.apache.org/jira/browse/HADOOP-14087) | S3A typo in pom.xml test exclusions | Major | fs/s3 | Aaron Fabbri | Aaron Fabbri | +| [HADOOP-14062](https://issues.apache.org/jira/browse/HADOOP-14062) | ApplicationMasterProtocolPBClientImpl.allocate fails with EOFException when RPC privacy is enabled | Critical | . | Steven Rand | Steven Rand | +| [HDFS-11431](https://issues.apache.org/jira/browse/HDFS-11431) | hadoop-hdfs-client JAR does not include ConfiguredFailoverProxyProvider | Blocker | build, hdfs-client | Steven Rand | Steven Rand | + + +### TESTS: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [MAPREDUCE-6191](https://issues.apache.org/jira/browse/MAPREDUCE-6191) | TestJavaSerialization fails with getting incorrect MR job result | Minor | test | sam liu | sam liu | +| [YARN-3339](https://issues.apache.org/jira/browse/YARN-3339) | TestDockerContainerExecutor should pull a single image and not the entire centos repository | Minor | test | Ravindra Kumar Naik | Ravindra Kumar Naik | +| [YARN-1880](https://issues.apache.org/jira/browse/YARN-1880) | Cleanup TestApplicationClientProtocolOnHA | Trivial | test | Tsuyoshi Ozawa | Tsuyoshi Ozawa | +| [HDFS-6263](https://issues.apache.org/jira/browse/HDFS-6263) | Remove DRFA.MaxBackupIndex config from log4j.properties | Minor | . | Akira Ajisaka | Abhiraj Butala | +| [HDFS-6408](https://issues.apache.org/jira/browse/HDFS-6408) | Remove redundant definitions in log4j.properties | Minor | test | Abhiraj Butala | Abhiraj Butala | +| [YARN-2666](https://issues.apache.org/jira/browse/YARN-2666) | TestFairScheduler.testContinuousScheduling fails Intermittently | Major | scheduler | Tsuyoshi Ozawa | zhihai xu | +| [HDFS-8247](https://issues.apache.org/jira/browse/HDFS-8247) | TestDiskspaceQuotaUpdate#testAppendOverTypeQuota is failing | Major | test | Anu Engineer | Xiaoyu Yao | +| [HADOOP-11881](https://issues.apache.org/jira/browse/HADOOP-11881) | test-patch.sh javac result is wildly wrong | Major | build, test | Allen Wittenauer | Kengo Seki | +| [HADOOP-11904](https://issues.apache.org/jira/browse/HADOOP-11904) | test-patch.sh goes into an infinite loop on non-maven builds | Critical | test | Allen Wittenauer | Allen Wittenauer | +| [YARN-3343](https://issues.apache.org/jira/browse/YARN-3343) | TestCapacitySchedulerNodeLabelUpdate.testNodeUpdate sometime fails in trunk | Minor | . | Xuan Gong | Rohith Sharma K S | +| [YARN-3580](https://issues.apache.org/jira/browse/YARN-3580) | [JDK 8] TestClientRMService.testGetLabelsToNodes fails | Major | test | Robert Kanter | Robert Kanter | +| [HDFS-7559](https://issues.apache.org/jira/browse/HDFS-7559) | Create unit test to automatically compare HDFS related classes and hdfs-default.xml | Minor | . | Ray Chiang | Ray Chiang | +| [HADOOP-11906](https://issues.apache.org/jira/browse/HADOOP-11906) | test-patch.sh should use 'file' command for patch determinism | Major | . | Allen Wittenauer | Sean Busbey | +| [YARN-3602](https://issues.apache.org/jira/browse/YARN-3602) | TestResourceLocalizationService.testPublicResourceInitializesLocalDir fails Intermittently due to IOException from cleanup | Minor | test | zhihai xu | zhihai xu | +| [HDFS-8243](https://issues.apache.org/jira/browse/HDFS-8243) | Files written by TestHostsFiles and TestNameNodeMXBean are causing Release Audit Warnings. | Minor | test | Ruth Wisniewski | Ruth Wisniewski | +| [HADOOP-11884](https://issues.apache.org/jira/browse/HADOOP-11884) | test-patch.sh should pull the real findbugs version | Minor | test | Allen Wittenauer | Kengo Seki | +| [HADOOP-11944](https://issues.apache.org/jira/browse/HADOOP-11944) | add option to test-patch to avoid relocating patch process directory | Minor | . | Sean Busbey | Sean Busbey | +| [HADOOP-11949](https://issues.apache.org/jira/browse/HADOOP-11949) | Add user-provided plugins to test-patch | Major | . | Sean Busbey | Sean Busbey | +| [HADOOP-12000](https://issues.apache.org/jira/browse/HADOOP-12000) | cannot use --java-home in test-patch | Major | scripts | Allen Wittenauer | Allen Wittenauer | +| [MAPREDUCE-6204](https://issues.apache.org/jira/browse/MAPREDUCE-6204) | TestJobCounters should use new properties instead of JobConf.MAPRED\_TASK\_JAVA\_OPTS | Minor | test | sam liu | sam liu | +| [HADOOP-12035](https://issues.apache.org/jira/browse/HADOOP-12035) | shellcheck plugin displays a wrong version potentially | Trivial | build | Kengo Seki | Kengo Seki | +| [HADOOP-12030](https://issues.apache.org/jira/browse/HADOOP-12030) | test-patch should only report on newly introduced findbugs warnings. | Major | . | Sean Busbey | Sean Busbey | +| [HADOOP-11930](https://issues.apache.org/jira/browse/HADOOP-11930) | test-patch in offline mode should tell maven to be in offline mode | Major | . | Sean Busbey | Sean Busbey | +| [HADOOP-11965](https://issues.apache.org/jira/browse/HADOOP-11965) | determine-flaky-tests needs a summary mode | Minor | . | Allen Wittenauer | Yufei Gu | +| [HDFS-8645](https://issues.apache.org/jira/browse/HDFS-8645) | Resolve inconsistent code in TestReplicationPolicy between trunk and branch-2 | Major | namenode | Zhe Zhang | Zhe Zhang | +| [YARN-2871](https://issues.apache.org/jira/browse/YARN-2871) | TestRMRestart#testRMRestartGetApplicationList sometime fails in trunk | Minor | . | Ted Yu | zhihai xu | +| [YARN-3956](https://issues.apache.org/jira/browse/YARN-3956) | Fix TestNodeManagerHardwareUtils fails on Mac | Minor | nodemanager | Varun Vasudev | Varun Vasudev | +| [HDFS-8834](https://issues.apache.org/jira/browse/HDFS-8834) | TestReplication#testReplicationWhenBlockCorruption is not valid after HDFS-6482 | Minor | datanode | Lei (Eddy) Xu | Lei (Eddy) Xu | +| [YARN-3992](https://issues.apache.org/jira/browse/YARN-3992) | TestApplicationPriority.testApplicationPriorityAllocation fails intermittently | Major | . | Zhijie Shen | Sunil G | +| [HDFS-2070](https://issues.apache.org/jira/browse/HDFS-2070) | Add more unit tests for FsShell getmerge | Major | test | XieXianshan | Daniel Templeton | +| [MAPREDUCE-5045](https://issues.apache.org/jira/browse/MAPREDUCE-5045) | UtilTest#isCygwin method appears to be unused | Trivial | contrib/streaming, test | Chris Nauroth | Neelesh Srinivas Salian | +| [YARN-3573](https://issues.apache.org/jira/browse/YARN-3573) | MiniMRYarnCluster constructor that starts the timeline server using a boolean should be marked deprecated | Major | timelineserver | Mit Desai | Brahma Reddy Battula | +| [HDFS-9295](https://issues.apache.org/jira/browse/HDFS-9295) | Add a thorough test of the full KMS code path | Critical | security, test | Daniel Templeton | Daniel Templeton | +| [HDFS-9339](https://issues.apache.org/jira/browse/HDFS-9339) | Extend full test of KMS ACLs | Major | test | Daniel Templeton | Daniel Templeton | +| [HDFS-9354](https://issues.apache.org/jira/browse/HDFS-9354) | Fix TestBalancer#testBalancerWithZeroThreadsForMove on Windows | Major | test | Xiaoyu Yao | Xiaoyu Yao | +| [HDFS-9410](https://issues.apache.org/jira/browse/HDFS-9410) | Some tests should always reset sysout and syserr | Minor | test | Xiao Chen | Xiao Chen | +| [HADOOP-12564](https://issues.apache.org/jira/browse/HADOOP-12564) | Upgrade JUnit3 TestCase to JUnit 4 in org.apache.hadoop.io package | Trivial | test | Dustin Cote | Dustin Cote | +| [HDFS-9153](https://issues.apache.org/jira/browse/HDFS-9153) | Pretty-format the output for DFSIO | Major | . | Kai Zheng | Kai Zheng | +| [HDFS-9429](https://issues.apache.org/jira/browse/HDFS-9429) | Tests in TestDFSAdminWithHA intermittently fail with EOFException | Major | test | Xiao Chen | Xiao Chen | +| [HADOOP-10729](https://issues.apache.org/jira/browse/HADOOP-10729) | Add tests for PB RPC in case version mismatch of client and server | Major | ipc | Junping Du | Junping Du | +| [HDFS-7553](https://issues.apache.org/jira/browse/HDFS-7553) | fix the TestDFSUpgradeWithHA due to BindException | Major | test | Liang Xie | Xiao Chen | +| [HDFS-9626](https://issues.apache.org/jira/browse/HDFS-9626) | TestBlockReplacement#testBlockReplacement fails occasionally | Minor | test | Xiao Chen | Xiao Chen | +| [HADOOP-12696](https://issues.apache.org/jira/browse/HADOOP-12696) | Add Tests for S3FileSystem Contract | Major | tools | Matthew Paduano | Matthew Paduano | +| [MAPREDUCE-6614](https://issues.apache.org/jira/browse/MAPREDUCE-6614) | Remove unnecessary code in TestMapreduceConfigFields | Minor | test | Akira Ajisaka | Kai Sasaki | +| [HADOOP-12736](https://issues.apache.org/jira/browse/HADOOP-12736) | TestTimedOutTestsListener#testThreadDumpAndDeadlocks sometimes times out | Major | . | Xiao Chen | Xiao Chen | +| [HADOOP-12715](https://issues.apache.org/jira/browse/HADOOP-12715) | TestValueQueue#testgetAtMostPolicyALL fails intermittently | Major | . | Xiao Chen | Xiao Chen | +| [HDFS-9773](https://issues.apache.org/jira/browse/HDFS-9773) | Remove dead code related to SimulatedFSDataset in tests | Minor | test | Akira Ajisaka | Brahma Reddy Battula | +| [HDFS-9888](https://issues.apache.org/jira/browse/HDFS-9888) | Allow reseting KerberosName in unit tests | Minor | . | Xiao Chen | Xiao Chen | +| [YARN-4717](https://issues.apache.org/jira/browse/YARN-4717) | TestResourceLocalizationService.testPublicResourceInitializesLocalDir fails Intermittently due to IllegalArgumentException from cleanup | Minor | nodemanager | Daniel Templeton | Daniel Templeton | +| [HDFS-9949](https://issues.apache.org/jira/browse/HDFS-9949) | Add a test case to ensure that the DataNode does not regenerate its UUID when a storage directory is cleared | Minor | . | Harsh J | Harsh J | +| [HADOOP-12738](https://issues.apache.org/jira/browse/HADOOP-12738) | Create unit test to automatically compare Common related classes and core-default.xml | Minor | . | Ray Chiang | Ray Chiang | +| [HADOOP-12101](https://issues.apache.org/jira/browse/HADOOP-12101) | Add automatic search of default Configuration variables to TestConfigurationFieldsBase | Major | test | Ray Chiang | Ray Chiang | +| [YARN-4947](https://issues.apache.org/jira/browse/YARN-4947) | Test timeout is happening for TestRMWebServicesNodes | Major | test | Bibin A Chundatt | Bibin A Chundatt | +| [YARN-5069](https://issues.apache.org/jira/browse/YARN-5069) | TestFifoScheduler.testResourceOverCommit race condition | Major | test | Eric Badger | Eric Badger | +| [YARN-5114](https://issues.apache.org/jira/browse/YARN-5114) | Add additional tests in TestRMWebServicesApps and rectify testInvalidAppAttempts failure in 2.8 | Minor | . | Bibin A Chundatt | Bibin A Chundatt | +| [HDFS-10375](https://issues.apache.org/jira/browse/HDFS-10375) | Remove redundant TestMiniDFSCluster.testDualClusters | Trivial | test | John Zhuge | Jiayi Zhou | +| [YARN-5023](https://issues.apache.org/jira/browse/YARN-5023) | TestAMRestart#testShouldNotCountFailureToMaxAttemptRetry random failure | Major | . | Bibin A Chundatt | sandflee | +| [YARN-5318](https://issues.apache.org/jira/browse/YARN-5318) | TestRMAdminService#testRefreshNodesResourceWithFileSystemBasedConfigurationProvider fails intermittently. | Minor | . | sandflee | Jun Gong | +| [YARN-5037](https://issues.apache.org/jira/browse/YARN-5037) | TestRMRestart#testQueueMetricsOnRMRestart random failure | Major | . | sandflee | sandflee | +| [YARN-5317](https://issues.apache.org/jira/browse/YARN-5317) | testAMRestartNotLostContainerCompleteMsg may fail | Minor | . | sandflee | sandflee | +| [YARN-5159](https://issues.apache.org/jira/browse/YARN-5159) | Wrong Javadoc tag in MiniYarnCluster | Major | documentation | Andras Bokor | Andras Bokor | +| [MAPREDUCE-6738](https://issues.apache.org/jira/browse/MAPREDUCE-6738) | TestJobListCache.testAddExisting failed intermittently in slow VM testbed | Minor | . | Junping Du | Junping Du | +| [YARN-5092](https://issues.apache.org/jira/browse/YARN-5092) | TestRMDelegationTokens fails intermittently | Major | test | Rohith Sharma K S | Jason Lowe | +| [HADOOP-10980](https://issues.apache.org/jira/browse/HADOOP-10980) | TestActiveStandbyElector fails occasionally in trunk | Minor | . | Ted Yu | Eric Badger | +| [HADOOP-13395](https://issues.apache.org/jira/browse/HADOOP-13395) | Enhance TestKMSAudit | Minor | kms | Xiao Chen | Xiao Chen | +| [YARN-5492](https://issues.apache.org/jira/browse/YARN-5492) | TestSubmitApplicationWithRMHA is failing sporadically during precommit builds | Major | test | Jason Lowe | Vrushali C | +| [YARN-5544](https://issues.apache.org/jira/browse/YARN-5544) | TestNodeBlacklistingOnAMFailures fails on trunk | Major | test | Varun Saxena | Sunil G | +| [HDFS-9745](https://issues.apache.org/jira/browse/HDFS-9745) | TestSecureNNWithQJM#testSecureMode sometimes fails with timeouts | Minor | . | Xiao Chen | Xiao Chen | +| [YARN-5389](https://issues.apache.org/jira/browse/YARN-5389) | TestYarnClient#testReservationDelete fails | Major | . | Rohith Sharma K S | Sean Po | +| [YARN-5560](https://issues.apache.org/jira/browse/YARN-5560) | Clean up bad exception catching practices in TestYarnClient | Major | . | Sean Po | Sean Po | +| [YARN-5608](https://issues.apache.org/jira/browse/YARN-5608) | TestAMRMClient.setup() fails with ArrayOutOfBoundsException | Major | test | Daniel Templeton | Daniel Templeton | +| [HDFS-10657](https://issues.apache.org/jira/browse/HDFS-10657) | testAclCLI.xml setfacl test should expect mask r-x | Minor | . | John Zhuge | John Zhuge | +| [HDFS-9333](https://issues.apache.org/jira/browse/HDFS-9333) | Some tests using MiniDFSCluster errored complaining port in use | Minor | test | Kai Zheng | Masatake Iwasaki | +| [HADOOP-13686](https://issues.apache.org/jira/browse/HADOOP-13686) | Adding additional unit test for Trash (I) | Major | . | Xiaoyu Yao | Weiwei Yang | +| [YARN-3568](https://issues.apache.org/jira/browse/YARN-3568) | TestAMRMTokens should use some random port | Major | test | Gera Shegalov | Takashi Ohnishi | +| [MAPREDUCE-6804](https://issues.apache.org/jira/browse/MAPREDUCE-6804) | Add timeout when starting JobHistoryServer in MiniMRYarnCluster | Minor | test | Andras Bokor | Andras Bokor | +| [HDFS-11272](https://issues.apache.org/jira/browse/HDFS-11272) | Refine the assert messages in TestFSDirAttrOp | Minor | test | Akira Ajisaka | Jimmy Xiang | +| [HDFS-11278](https://issues.apache.org/jira/browse/HDFS-11278) | Add missing @Test annotation for TestSafeMode.testSafeModeUtils() | Trivial | namenode | Lukas Majercak | Lukas Majercak | + + +### SUB-TASKS: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [YARN-3356](https://issues.apache.org/jira/browse/YARN-3356) | Capacity Scheduler FiCaSchedulerApp should use ResourceUsage to track used-resources-by-label. | Major | capacityscheduler, resourcemanager | Wangda Tan | Wangda Tan | +| [YARN-3345](https://issues.apache.org/jira/browse/YARN-3345) | Add non-exclusive node label API to RMAdmin protocol and NodeLabelsManager | Major | api, client, resourcemanager | Wangda Tan | Wangda Tan | +| [HDFS-7854](https://issues.apache.org/jira/browse/HDFS-7854) | Separate class DataStreamer out of DFSOutputStream | Major | hdfs-client | Li Bo | Li Bo | +| [HDFS-7713](https://issues.apache.org/jira/browse/HDFS-7713) | Implement mkdirs in the HDFS Web UI | Major | ui | Ravi Prakash | Ravi Prakash | +| [YARN-2495](https://issues.apache.org/jira/browse/YARN-2495) | Allow admin specify labels from each NM (Distributed configuration) | Major | resourcemanager | Wangda Tan | Naganarasimha G R | +| [HDFS-7893](https://issues.apache.org/jira/browse/HDFS-7893) | Update the POM to create a separate hdfs-client jar | Major | build | Haohui Mai | Haohui Mai | +| [YARN-3365](https://issues.apache.org/jira/browse/YARN-3365) | Add support for using the 'tc' tool via container-executor | Major | nodemanager | Sidharta Seethana | Sidharta Seethana | +| [HDFS-8034](https://issues.apache.org/jira/browse/HDFS-8034) | Fix TestDFSClientRetries#testDFSClientConfigurationLocateFollowingBlockInitialDelay for Windows | Minor | test | Xiaoyu Yao | Xiaoyu Yao | +| [YARN-3110](https://issues.apache.org/jira/browse/YARN-3110) | Few issues in ApplicationHistory web ui | Minor | applications, timelineserver | Bibin A Chundatt | Naganarasimha G R | +| [HDFS-8049](https://issues.apache.org/jira/browse/HDFS-8049) | Annotation client implementation as private | Major | hdfs-client | Tsz Wo Nicholas Sze | Takuya Fukudome | +| [HDFS-8079](https://issues.apache.org/jira/browse/HDFS-8079) | Separate the client retry conf from DFSConfigKeys | Major | hdfs-client | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze | +| [HDFS-8080](https://issues.apache.org/jira/browse/HDFS-8080) | Separate JSON related routines used by WebHdfsFileSystem to a package local class | Minor | hdfs-client | Haohui Mai | Haohui Mai | +| [HDFS-8085](https://issues.apache.org/jira/browse/HDFS-8085) | Move CorruptFileBlockIterator to the hdfs.client.impl package | Minor | hdfs-client | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze | +| [HDFS-8089](https://issues.apache.org/jira/browse/HDFS-8089) | Move o.a.h.hdfs.web.resources.\* to the client jars | Minor | build | Haohui Mai | Haohui Mai | +| [HDFS-8102](https://issues.apache.org/jira/browse/HDFS-8102) | Separate webhdfs retry configuration keys from DFSConfigKeys | Minor | hdfs-client | Haohui Mai | Haohui Mai | +| [YARN-1376](https://issues.apache.org/jira/browse/YARN-1376) | NM need to notify the log aggregation status to RM through Node heartbeat | Major | . | Xuan Gong | Xuan Gong | +| [HDFS-8100](https://issues.apache.org/jira/browse/HDFS-8100) | Refactor DFSClient.Conf to a standalone class | Minor | hdfs-client | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze | +| [HDFS-8103](https://issues.apache.org/jira/browse/HDFS-8103) | Move BlockTokenSecretManager.AccessMode into BlockTokenIdentifier | Minor | security | Haohui Mai | Haohui Mai | +| [HDFS-8084](https://issues.apache.org/jira/browse/HDFS-8084) | Separate the client failover conf from DFSConfigKeys | Major | hdfs-client | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze | +| [HDFS-8083](https://issues.apache.org/jira/browse/HDFS-8083) | Separate the client write conf from DFSConfigKeys | Major | hdfs-client | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze | +| [YARN-3347](https://issues.apache.org/jira/browse/YARN-3347) | Improve YARN log command to get AMContainer logs as well as running containers logs | Major | log-aggregation | Xuan Gong | Xuan Gong | +| [YARN-3443](https://issues.apache.org/jira/browse/YARN-3443) | Create a 'ResourceHandler' subsystem to ease addition of support for new resource types on the NM | Major | nodemanager | Sidharta Seethana | Sidharta Seethana | +| [HDFS-7701](https://issues.apache.org/jira/browse/HDFS-7701) | Support reporting per storage type quota and usage with hadoop/hdfs shell | Major | datanode, namenode | Xiaoyu Yao | Peter Shi | +| [YARN-3361](https://issues.apache.org/jira/browse/YARN-3361) | CapacityScheduler side changes to support non-exclusive node labels | Major | capacityscheduler | Wangda Tan | Wangda Tan | +| [YARN-3318](https://issues.apache.org/jira/browse/YARN-3318) | Create Initial OrderingPolicy Framework and FifoOrderingPolicy | Major | scheduler | Craig Welch | Craig Welch | +| [YARN-3326](https://issues.apache.org/jira/browse/YARN-3326) | Support RESTful API for getLabelsToNodes | Minor | resourcemanager | Naganarasimha G R | Naganarasimha G R | +| [YARN-3354](https://issues.apache.org/jira/browse/YARN-3354) | Container should contains node-labels asked by original ResourceRequests | Major | api, capacityscheduler, nodemanager, resourcemanager | Wangda Tan | Wangda Tan | +| [HDFS-8082](https://issues.apache.org/jira/browse/HDFS-8082) | Separate the client read conf from DFSConfigKeys | Major | hdfs-client | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze | +| [HDFS-8165](https://issues.apache.org/jira/browse/HDFS-8165) | Move GRANDFATHER\_GENERATION\_STAMP and GRANDFATER\_INODE\_ID to hdfs-client | Major | build | Haohui Mai | Haohui Mai | +| [YARN-1402](https://issues.apache.org/jira/browse/YARN-1402) | Related Web UI, CLI changes on exposing client API to check log aggregation status | Major | . | Xuan Gong | Xuan Gong | +| [YARN-2696](https://issues.apache.org/jira/browse/YARN-2696) | Queue sorting in CapacityScheduler should consider node label | Major | capacityscheduler, resourcemanager | Wangda Tan | Wangda Tan | +| [YARN-3487](https://issues.apache.org/jira/browse/YARN-3487) | CapacityScheduler scheduler lock obtained unnecessarily when calling getQueue | Critical | capacityscheduler | Jason Lowe | Jason Lowe | +| [YARN-3136](https://issues.apache.org/jira/browse/YARN-3136) | getTransferredContainers can be a bottleneck during AM registration | Major | scheduler | Jason Lowe | Sunil G | +| [HDFS-8169](https://issues.apache.org/jira/browse/HDFS-8169) | Move LocatedBlocks and related classes to hdfs-client | Major | build, hdfs-client | Haohui Mai | Haohui Mai | +| [YARN-3463](https://issues.apache.org/jira/browse/YARN-3463) | Integrate OrderingPolicy Framework with CapacityScheduler | Major | capacityscheduler | Craig Welch | Craig Welch | +| [HDFS-8185](https://issues.apache.org/jira/browse/HDFS-8185) | Separate client related routines in HAUtil into a new class | Major | build, hdfs-client | Haohui Mai | Haohui Mai | +| [YARN-3225](https://issues.apache.org/jira/browse/YARN-3225) | New parameter or CLI for decommissioning node gracefully in RMAdmin CLI | Major | graceful | Junping Du | Devaraj K | +| [HDFS-8218](https://issues.apache.org/jira/browse/HDFS-8218) | Move classes that used by ClientProtocol into hdfs-client | Major | build | Haohui Mai | Haohui Mai | +| [YARN-3366](https://issues.apache.org/jira/browse/YARN-3366) | Outbound network bandwidth : classify/shape traffic originating from YARN containers | Major | . | Sidharta Seethana | Sidharta Seethana | +| [YARN-2605](https://issues.apache.org/jira/browse/YARN-2605) | [RM HA] Rest api endpoints doing redirect incorrectly | Major | resourcemanager | bc Wong | Xuan Gong | +| [YARN-3319](https://issues.apache.org/jira/browse/YARN-3319) | Implement a FairOrderingPolicy | Major | scheduler | Craig Welch | Craig Welch | +| [YARN-3413](https://issues.apache.org/jira/browse/YARN-3413) | Node label attributes (like exclusivity) should settable via addToClusterNodeLabels but shouldn't be changeable at runtime | Major | api, client, resourcemanager | Wangda Tan | Wangda Tan | +| [YARN-2498](https://issues.apache.org/jira/browse/YARN-2498) | Respect labels in preemption policy of capacity scheduler for inter-queue preemption | Major | resourcemanager | Wangda Tan | Wangda Tan | +| [YARN-2740](https://issues.apache.org/jira/browse/YARN-2740) | Fix NodeLabelsManager to properly handle node label modifications when distributed node label configuration enabled | Major | resourcemanager | Wangda Tan | Naganarasimha G R | +| [YARN-3544](https://issues.apache.org/jira/browse/YARN-3544) | AM logs link missing in the RM UI for a completed app | Blocker | . | Hitesh Shah | Xuan Gong | +| [YARN-2619](https://issues.apache.org/jira/browse/YARN-2619) | NodeManager: Add cgroups support for disk I/O isolation | Major | . | Wei Yan | Varun Vasudev | +| [HDFS-8086](https://issues.apache.org/jira/browse/HDFS-8086) | Move LeaseRenewer to the hdfs.client.impl package | Minor | hdfs-client | Tsz Wo Nicholas Sze | Takanobu Asanuma | +| [YARN-3006](https://issues.apache.org/jira/browse/YARN-3006) | Improve the error message when attempting manual failover with auto-failover enabled | Minor | . | Akira Ajisaka | Akira Ajisaka | +| [HDFS-8249](https://issues.apache.org/jira/browse/HDFS-8249) | Separate HdfsConstants into the client and the server side class | Major | hdfs-client | Haohui Mai | Haohui Mai | +| [HDFS-8309](https://issues.apache.org/jira/browse/HDFS-8309) | Skip unit test using DataNodeTestUtils#injectDataDirFailure() on Windows | Minor | test | Xiaoyu Yao | Xiaoyu Yao | +| [HDFS-8237](https://issues.apache.org/jira/browse/HDFS-8237) | Move all protocol classes used by ClientProtocol to hdfs-client | Major | build | Haohui Mai | Haohui Mai | +| [HDFS-8314](https://issues.apache.org/jira/browse/HDFS-8314) | Move HdfsServerConstants#IO\_FILE\_BUFFER\_SIZE and SMALL\_BUFFER\_SIZE to the users | Major | . | Haohui Mai | Li Lu | +| [HDFS-8310](https://issues.apache.org/jira/browse/HDFS-8310) | Fix TestCLI.testAll 'help: help for find' on Windows | Minor | test | Xiaoyu Yao | Kiran Kumar M R | +| [YARN-3301](https://issues.apache.org/jira/browse/YARN-3301) | Fix the format issue of the new RM web UI and AHS web UI after YARN-3272 / YARN-3262 | Major | resourcemanager | Xuan Gong | Xuan Gong | +| [YARN-3448](https://issues.apache.org/jira/browse/YARN-3448) | Add Rolling Time To Lives Level DB Plugin Capabilities | Major | timelineserver | Jonathan Eagles | Jonathan Eagles | +| [YARN-2918](https://issues.apache.org/jira/browse/YARN-2918) | Don't fail RM if queue's configured labels are not existed in cluster-node-labels | Major | resourcemanager | Rohith Sharma K S | Wangda Tan | +| [YARN-644](https://issues.apache.org/jira/browse/YARN-644) | Basic null check is not performed on passed in arguments before using them in ContainerManagerImpl.startContainer | Minor | nodemanager | Omkar Vinit Joshi | Varun Saxena | +| [YARN-3593](https://issues.apache.org/jira/browse/YARN-3593) | Add label-type and Improve "DEFAULT\_PARTITION" in Node Labels Page | Major | webapp | Naganarasimha G R | Naganarasimha G R | +| [YARN-2331](https://issues.apache.org/jira/browse/YARN-2331) | Distinguish shutdown during supervision vs. shutdown for rolling upgrade | Major | nodemanager | Jason Lowe | Jason Lowe | +| [YARN-3579](https://issues.apache.org/jira/browse/YARN-3579) | CommonNodeLabelsManager should support NodeLabel instead of string label name when getting node-to-label/label-to-label mappings | Minor | resourcemanager | Sunil G | Sunil G | +| [YARN-3505](https://issues.apache.org/jira/browse/YARN-3505) | Node's Log Aggregation Report with SUCCEED should not cached in RMApps | Critical | log-aggregation | Junping Du | Xuan Gong | +| [HDFS-8403](https://issues.apache.org/jira/browse/HDFS-8403) | Eliminate retries in TestFileCreation#testOverwriteOpenForWrite | Major | test | Arpit Agarwal | Arpit Agarwal | +| [HDFS-8157](https://issues.apache.org/jira/browse/HDFS-8157) | Writes to RAM DISK reserve locked memory for block files | Major | datanode | Arpit Agarwal | Arpit Agarwal | +| [YARN-3541](https://issues.apache.org/jira/browse/YARN-3541) | Add version info on timeline service / generic history web UI and REST API | Major | timelineserver | Zhijie Shen | Zhijie Shen | +| [YARN-3565](https://issues.apache.org/jira/browse/YARN-3565) | NodeHeartbeatRequest/RegisterNodeManagerRequest should use NodeLabel object instead of String | Blocker | api, client, resourcemanager | Wangda Tan | Naganarasimha G R | +| [YARN-3583](https://issues.apache.org/jira/browse/YARN-3583) | Support of NodeLabel object instead of plain String in YarnClient side. | Major | client | Sunil G | Sunil G | +| [YARN-3609](https://issues.apache.org/jira/browse/YARN-3609) | Move load labels from storage from serviceInit to serviceStart to make it works with RM HA case. | Major | resourcemanager | Wangda Tan | Wangda Tan | +| [YARN-3684](https://issues.apache.org/jira/browse/YARN-3684) | Change ContainerExecutor's primary lifecycle methods to use a more extensible mechanism for passing information. | Major | yarn | Sidharta Seethana | Sidharta Seethana | +| [HDFS-8454](https://issues.apache.org/jira/browse/HDFS-8454) | Remove unnecessary throttling in TestDatanodeDeath | Major | test | Arpit Agarwal | Arpit Agarwal | +| [YARN-3632](https://issues.apache.org/jira/browse/YARN-3632) | Ordering policy should be allowed to reorder an application when demand changes | Major | capacityscheduler | Craig Welch | Craig Welch | +| [YARN-3686](https://issues.apache.org/jira/browse/YARN-3686) | CapacityScheduler should trim default\_node\_label\_expression | Critical | api, client, resourcemanager | Wangda Tan | Sunil G | +| [YARN-3647](https://issues.apache.org/jira/browse/YARN-3647) | RMWebServices api's should use updated api from CommonNodeLabelsManager to get NodeLabel object | Major | resourcemanager | Sunil G | Sunil G | +| [YARN-3581](https://issues.apache.org/jira/browse/YARN-3581) | Deprecate -directlyAccessNodeLabelStore in RMAdminCLI | Major | api, client, resourcemanager | Wangda Tan | Naganarasimha G R | +| [HDFS-8482](https://issues.apache.org/jira/browse/HDFS-8482) | Rename BlockInfoContiguous to BlockInfo | Major | . | Zhe Zhang | Zhe Zhang | +| [YARN-3700](https://issues.apache.org/jira/browse/YARN-3700) | ATS Web Performance issue at load time when large number of jobs | Major | resourcemanager, webapp, yarn | Xuan Gong | Xuan Gong | +| [YARN-3716](https://issues.apache.org/jira/browse/YARN-3716) | Node-label-expression should be included by ResourceRequestPBImpl.toString | Minor | api | Xianyin Xin | Xianyin Xin | +| [YARN-3740](https://issues.apache.org/jira/browse/YARN-3740) | Fixed the typo with the configuration name: APPLICATION\_HISTORY\_PREFIX\_MAX\_APPS | Major | resourcemanager, webapp, yarn | Xuan Gong | Xuan Gong | +| [YARN-2900](https://issues.apache.org/jira/browse/YARN-2900) | Application (Attempt and Container) Not Found in AHS results in Internal Server Error (500) | Major | timelineserver | Jonathan Eagles | Mit Desai | +| [HDFS-8489](https://issues.apache.org/jira/browse/HDFS-8489) | Subclass BlockInfo to represent contiguous blocks | Major | namenode | Zhe Zhang | Zhe Zhang | +| [YARN-2392](https://issues.apache.org/jira/browse/YARN-2392) | add more diags about app retry limits on AM failures | Minor | resourcemanager | Steve Loughran | Steve Loughran | +| [YARN-3766](https://issues.apache.org/jira/browse/YARN-3766) | ATS Web UI breaks because of YARN-3467 | Blocker | resourcemanager, webapp, yarn | Xuan Gong | Xuan Gong | +| [YARN-1462](https://issues.apache.org/jira/browse/YARN-1462) | AHS API and other AHS changes to handle tags for completed MR jobs | Major | . | Karthik Kambatla | Xuan Gong | +| [YARN-3787](https://issues.apache.org/jira/browse/YARN-3787) | loading applications by filtering appstartedTime period for ATS Web UI | Major | resourcemanager, webapp, yarn | Xuan Gong | Xuan Gong | +| [HDFS-7923](https://issues.apache.org/jira/browse/HDFS-7923) | The DataNodes should rate-limit their full block reports by asking the NN on heartbeat messages | Major | . | Colin P. McCabe | Colin P. McCabe | +| [HDFS-8540](https://issues.apache.org/jira/browse/HDFS-8540) | Mover should exit with NO\_MOVE\_BLOCK if no block can be moved | Major | balancer & mover | Tsz Wo Nicholas Sze | Surendra Singh Lilhore | +| [YARN-3711](https://issues.apache.org/jira/browse/YARN-3711) | Documentation of ResourceManager HA should explain configurations about listen addresses | Minor | documentation | Masatake Iwasaki | Masatake Iwasaki | +| [HDFS-8597](https://issues.apache.org/jira/browse/HDFS-8597) | Fix TestFSImage#testZeroBlockSize on Windows | Major | datanode, test | Xiaoyu Yao | Xiaoyu Yao | +| [HDFS-7164](https://issues.apache.org/jira/browse/HDFS-7164) | Feature documentation for HDFS-6581 | Major | documentation | Arpit Agarwal | Arpit Agarwal | +| [HDFS-8238](https://issues.apache.org/jira/browse/HDFS-8238) | Move ClientProtocol to the hdfs-client | Major | build | Haohui Mai | Takanobu Asanuma | +| [HDFS-6249](https://issues.apache.org/jira/browse/HDFS-6249) | Output AclEntry in PBImageXmlWriter | Minor | tools | Akira Ajisaka | Surendra Singh Lilhore | +| [YARN-3521](https://issues.apache.org/jira/browse/YARN-3521) | Support return structured NodeLabel objects in REST API | Major | api, client, resourcemanager | Wangda Tan | Sunil G | +| [HDFS-8192](https://issues.apache.org/jira/browse/HDFS-8192) | Eviction should key off used locked memory instead of ram disk free space | Major | datanode | Arpit Agarwal | Arpit Agarwal | +| [HDFS-8651](https://issues.apache.org/jira/browse/HDFS-8651) | Make hadoop-hdfs-project Native code -Wall-clean | Major | native | Alan Burlison | Alan Burlison | +| [HADOOP-12036](https://issues.apache.org/jira/browse/HADOOP-12036) | Consolidate all of the cmake extensions in one directory | Major | . | Allen Wittenauer | Alan Burlison | +| [HDFS-7390](https://issues.apache.org/jira/browse/HDFS-7390) | Provide JMX metrics per storage type | Major | . | Benoy Antony | Benoy Antony | +| [HADOOP-12104](https://issues.apache.org/jira/browse/HADOOP-12104) | Migrate Hadoop Pipes native build to new CMake framework | Major | build | Alan Burlison | Alan Burlison | +| [HADOOP-12112](https://issues.apache.org/jira/browse/HADOOP-12112) | Make hadoop-common-project Native code -Wall-clean | Major | native | Alan Burlison | Alan Burlison | +| [HDFS-8493](https://issues.apache.org/jira/browse/HDFS-8493) | Consolidate truncate() related implementation in a single class | Major | . | Haohui Mai | Rakesh R | +| [HDFS-8635](https://issues.apache.org/jira/browse/HDFS-8635) | Migrate HDFS native build to new CMake framework | Major | build | Alan Burlison | Alan Burlison | +| [YARN-3827](https://issues.apache.org/jira/browse/YARN-3827) | Migrate YARN native build to new CMake framework | Major | build | Alan Burlison | Alan Burlison | +| [MAPREDUCE-6376](https://issues.apache.org/jira/browse/MAPREDUCE-6376) | Add avro binary support for jhist files | Major | jobhistoryserver | Ray Chiang | Ray Chiang | +| [HDFS-8620](https://issues.apache.org/jira/browse/HDFS-8620) | Clean up the checkstyle warinings about ClientProtocol | Major | . | Takanobu Asanuma | Takanobu Asanuma | +| [HDFS-8726](https://issues.apache.org/jira/browse/HDFS-8726) | Move protobuf files that define the client-sever protocols to hdfs-client | Major | build | Haohui Mai | Haohui Mai | +| [YARN-1012](https://issues.apache.org/jira/browse/YARN-1012) | Report NM aggregated container resource utilization in heartbeat | Major | nodemanager | Arun C Murthy | Inigo Goiri | +| [YARN-3800](https://issues.apache.org/jira/browse/YARN-3800) | Reduce storage footprint for ReservationAllocation | Major | capacityscheduler, fairscheduler, resourcemanager | Anubhav Dhoot | Anubhav Dhoot | +| [YARN-3445](https://issues.apache.org/jira/browse/YARN-3445) | Cache runningApps in RMNode for getting running apps on given NodeId | Major | nodemanager, resourcemanager | Junping Du | Junping Du | +| [YARN-3116](https://issues.apache.org/jira/browse/YARN-3116) | [Collector wireup] We need an assured way to determine if a container is an AM container on NM | Major | nodemanager, timelineserver | Zhijie Shen | Giovanni Matteo Fumarola | +| [HDFS-8541](https://issues.apache.org/jira/browse/HDFS-8541) | Mover should exit with NO\_MOVE\_PROGRESS if there is no move progress | Minor | balancer & mover | Tsz Wo Nicholas Sze | Surendra Singh Lilhore | +| [HDFS-8742](https://issues.apache.org/jira/browse/HDFS-8742) | Inotify: Support event for OP\_TRUNCATE | Major | namenode | Surendra Singh Lilhore | Surendra Singh Lilhore | +| [YARN-1449](https://issues.apache.org/jira/browse/YARN-1449) | AM-NM protocol changes to support container resizing | Major | api | Wangda Tan (No longer used) | MENG DING | +| [HADOOP-11974](https://issues.apache.org/jira/browse/HADOOP-11974) | Fix FIONREAD #include on Solaris | Minor | net | Alan Burlison | Alan Burlison | +| [YARN-3930](https://issues.apache.org/jira/browse/YARN-3930) | FileSystemNodeLabelsStore should make sure edit log file closed when exception is thrown | Major | api, client, resourcemanager | Dian Fu | Dian Fu | +| [YARN-3844](https://issues.apache.org/jira/browse/YARN-3844) | Make hadoop-yarn-project Native code -Wall-clean | Major | build | Alan Burlison | Alan Burlison | +| [HDFS-8794](https://issues.apache.org/jira/browse/HDFS-8794) | Improve CorruptReplicasMap#corruptReplicasMap | Major | . | Yi Liu | Yi Liu | +| [HDFS-7483](https://issues.apache.org/jira/browse/HDFS-7483) | Display information per tier on the Namenode UI | Major | . | Benoy Antony | Benoy Antony | +| [YARN-2003](https://issues.apache.org/jira/browse/YARN-2003) | Support for Application priority : Changes in RM and Capacity Scheduler | Major | resourcemanager | Sunil G | Sunil G | +| [HDFS-8721](https://issues.apache.org/jira/browse/HDFS-8721) | Add a metric for number of encryption zones | Major | encryption | Rakesh R | Rakesh R | +| [YARN-1645](https://issues.apache.org/jira/browse/YARN-1645) | ContainerManager implementation to support container resizing | Major | nodemanager | Wangda Tan | MENG DING | +| [HDFS-8495](https://issues.apache.org/jira/browse/HDFS-8495) | Consolidate append() related implementation into a single class | Major | namenode | Rakesh R | Rakesh R | +| [HDFS-8795](https://issues.apache.org/jira/browse/HDFS-8795) | Improve InvalidateBlocks#node2blocks | Major | . | Yi Liu | Yi Liu | +| [HADOOP-12184](https://issues.apache.org/jira/browse/HADOOP-12184) | Remove unused Linux-specific constants in NativeIO | Major | native | Martin Walsh | Martin Walsh | +| [HDFS-8730](https://issues.apache.org/jira/browse/HDFS-8730) | Clean up the import statements in ClientProtocol | Minor | . | Takanobu Asanuma | Takanobu Asanuma | +| [YARN-3969](https://issues.apache.org/jira/browse/YARN-3969) | Allow jobs to be submitted to reservation that is active but does not have any allocations | Major | capacityscheduler, fairscheduler, resourcemanager | Subru Krishnan | Subru Krishnan | +| [HADOOP-12170](https://issues.apache.org/jira/browse/HADOOP-12170) | hadoop-common's JNIFlags.cmake is redundant and can be removed | Minor | native | Alan Burlison | Alan Burlison | +| [YARN-3656](https://issues.apache.org/jira/browse/YARN-3656) | LowCost: A Cost-Based Placement Agent for YARN Reservations | Major | capacityscheduler, resourcemanager | Ishai Menache | Jonathan Yaniv | +| [YARN-3852](https://issues.apache.org/jira/browse/YARN-3852) | Add docker container support to container-executor | Major | yarn | Sidharta Seethana | Abin Shahab | +| [YARN-3853](https://issues.apache.org/jira/browse/YARN-3853) | Add docker container runtime support to LinuxContainterExecutor | Major | yarn | Sidharta Seethana | Sidharta Seethana | +| [YARN-3867](https://issues.apache.org/jira/browse/YARN-3867) | ContainerImpl changes to support container resizing | Major | nodemanager | MENG DING | MENG DING | +| [HDFS-7192](https://issues.apache.org/jira/browse/HDFS-7192) | DN should ignore lazyPersist hint if the writer is not local | Major | datanode | Arpit Agarwal | Arpit Agarwal | +| [YARN-433](https://issues.apache.org/jira/browse/YARN-433) | When RM is catching up with node updates then it should not expire acquired containers | Major | resourcemanager | Bikas Saha | Xuan Gong | +| [MAPREDUCE-6394](https://issues.apache.org/jira/browse/MAPREDUCE-6394) | Speed up Task processing loop in HsTasksBlock#render() | Major | jobhistoryserver | Ray Chiang | Ray Chiang | +| [HADOOP-7824](https://issues.apache.org/jira/browse/HADOOP-7824) | NativeIO.java flags and identifiers must be set correctly for each platform, not hardcoded to their Linux values | Major | native | Dmytro Shteflyuk | Martin Walsh | +| [YARN-3543](https://issues.apache.org/jira/browse/YARN-3543) | ApplicationReport should be able to tell whether the Application is AM managed or not. | Major | api | Spandan Dutta | Rohith Sharma K S | +| [YARN-4004](https://issues.apache.org/jira/browse/YARN-4004) | container-executor should print output of docker logs if the docker container exits with non-0 exit status | Major | nodemanager | Varun Vasudev | Varun Vasudev | +| [YARN-3736](https://issues.apache.org/jira/browse/YARN-3736) | Add RMStateStore apis to store and load accepted reservations for failover | Major | capacityscheduler, fairscheduler, resourcemanager | Subru Krishnan | Anubhav Dhoot | +| [YARN-1643](https://issues.apache.org/jira/browse/YARN-1643) | Make ContainersMonitor can support change monitoring size of an allocated container in NM side | Major | nodemanager | Wangda Tan | MENG DING | +| [YARN-3974](https://issues.apache.org/jira/browse/YARN-3974) | Refactor the reservation system test cases to use parameterized base test | Major | capacityscheduler, fairscheduler | Subru Krishnan | Subru Krishnan | +| [YARN-3948](https://issues.apache.org/jira/browse/YARN-3948) | Display Application Priority in RM Web UI | Major | webapp | Sunil G | Sunil G | +| [YARN-3873](https://issues.apache.org/jira/browse/YARN-3873) | pendingApplications in LeafQueue should also use OrderingPolicy | Major | capacityscheduler | Sunil G | Sunil G | +| [YARN-3887](https://issues.apache.org/jira/browse/YARN-3887) | Support for changing Application priority during runtime | Major | capacityscheduler, resourcemanager | Sunil G | Sunil G | +| [HDFS-8805](https://issues.apache.org/jira/browse/HDFS-8805) | Archival Storage: getStoragePolicy should not need superuser privilege | Major | balancer & mover, namenode | Hui Zheng | Brahma Reddy Battula | +| [HDFS-8052](https://issues.apache.org/jira/browse/HDFS-8052) | Move WebHdfsFileSystem into hadoop-hdfs-client | Major | build | Haohui Mai | Haohui Mai | +| [YARN-4023](https://issues.apache.org/jira/browse/YARN-4023) | Publish Application Priority to TimelineServer | Major | timelineserver | Sunil G | Sunil G | +| [HDFS-8824](https://issues.apache.org/jira/browse/HDFS-8824) | Do not use small blocks for balancing the cluster | Major | balancer & mover | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze | +| [YARN-3534](https://issues.apache.org/jira/browse/YARN-3534) | Collect memory/cpu usage on the node | Major | nodemanager, resourcemanager | Inigo Goiri | Inigo Goiri | +| [HDFS-8801](https://issues.apache.org/jira/browse/HDFS-8801) | Convert BlockInfoUnderConstruction as a feature | Major | namenode | Zhe Zhang | Jing Zhao | +| [HDFS-8792](https://issues.apache.org/jira/browse/HDFS-8792) | BlockManager#postponedMisreplicatedBlocks should use a LightWeightHashSet to save memory | Major | . | Yi Liu | Yi Liu | +| [HDFS-8862](https://issues.apache.org/jira/browse/HDFS-8862) | BlockManager#excessReplicateMap should use a HashMap | Major | namenode | Yi Liu | Yi Liu | +| [HDFS-8278](https://issues.apache.org/jira/browse/HDFS-8278) | HDFS Balancer should consider remaining storage % when checking for under-utilized machines | Major | balancer & mover | Gopal V | Tsz Wo Nicholas Sze | +| [HDFS-8826](https://issues.apache.org/jira/browse/HDFS-8826) | Balancer may not move blocks efficiently in some cases | Major | balancer & mover | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze | +| [HDFS-8803](https://issues.apache.org/jira/browse/HDFS-8803) | Move DfsClientConf to hdfs-client | Major | build | Haohui Mai | Mingliang Liu | +| [YARN-2923](https://issues.apache.org/jira/browse/YARN-2923) | Support configuration based NodeLabelsProvider Service in Distributed Node Label Configuration Setup | Major | nodemanager | Naganarasimha G R | Naganarasimha G R | +| [YARN-1644](https://issues.apache.org/jira/browse/YARN-1644) | RM-NM protocol changes and NodeStatusUpdater implementation to support container resizing | Major | nodemanager | Wangda Tan | MENG DING | +| [YARN-3868](https://issues.apache.org/jira/browse/YARN-3868) | ContainerManager recovery for container resizing | Major | nodemanager | MENG DING | MENG DING | +| [HDFS-8823](https://issues.apache.org/jira/browse/HDFS-8823) | Move replication factor into individual blocks | Major | . | Haohui Mai | Haohui Mai | +| [YARN-221](https://issues.apache.org/jira/browse/YARN-221) | NM should provide a way for AM to tell it not to aggregate logs. | Major | log-aggregation, nodemanager | Robert Joseph Evans | Ming Ma | +| [HDFS-8934](https://issues.apache.org/jira/browse/HDFS-8934) | Move ShortCircuitShm to hdfs-client | Major | build | Mingliang Liu | Mingliang Liu | +| [HDFS-8948](https://issues.apache.org/jira/browse/HDFS-8948) | Use GenericTestUtils to set log levels in TestPread and TestReplaceDatanodeOnFailure | Major | build | Mingliang Liu | Mingliang Liu | +| [YARN-4014](https://issues.apache.org/jira/browse/YARN-4014) | Support user cli interface in for Application Priority | Major | client, resourcemanager | Rohith Sharma K S | Rohith Sharma K S | +| [HDFS-8951](https://issues.apache.org/jira/browse/HDFS-8951) | Move the shortcircuit package to hdfs-client | Major | build | Mingliang Liu | Mingliang Liu | +| [HDFS-8248](https://issues.apache.org/jira/browse/HDFS-8248) | Store INodeId instead of the INodeFile object in BlockInfoContiguous | Major | . | Haohui Mai | Haohui Mai | +| [HDFS-8962](https://issues.apache.org/jira/browse/HDFS-8962) | Clean up checkstyle warnings in o.a.h.hdfs.DfsClientConf | Major | build | Mingliang Liu | Mingliang Liu | +| [YARN-3250](https://issues.apache.org/jira/browse/YARN-3250) | Support admin cli interface in for Application Priority | Major | resourcemanager | Sunil G | Rohith Sharma K S | +| [HDFS-8925](https://issues.apache.org/jira/browse/HDFS-8925) | Move BlockReaderLocal to hdfs-client | Major | build | Mingliang Liu | Mingliang Liu | +| [HDFS-8980](https://issues.apache.org/jira/browse/HDFS-8980) | Remove unnecessary block replacement in INodeFile | Major | namenode | Jing Zhao | Jing Zhao | +| [HDFS-8990](https://issues.apache.org/jira/browse/HDFS-8990) | Move RemoteBlockReader to hdfs-client module | Major | build | Mingliang Liu | Mingliang Liu | +| [YARN-4092](https://issues.apache.org/jira/browse/YARN-4092) | RM HA UI redirection needs to be fixed when both RMs are in standby mode | Major | resourcemanager | Xuan Gong | Xuan Gong | +| [YARN-4082](https://issues.apache.org/jira/browse/YARN-4082) | Container shouldn't be killed when node's label updated. | Major | capacity scheduler | Wangda Tan | Wangda Tan | +| [YARN-2801](https://issues.apache.org/jira/browse/YARN-2801) | Add documentation for node labels feature | Major | documentation | Gururaj Shetty | Wangda Tan | +| [YARN-3893](https://issues.apache.org/jira/browse/YARN-3893) | Both RM in active state when Admin#transitionToActive failure from refeshAll() | Critical | resourcemanager | Bibin A Chundatt | Bibin A Chundatt | +| [HDFS-8890](https://issues.apache.org/jira/browse/HDFS-8890) | Allow admin to specify which blockpools the balancer should run on | Major | balancer & mover | Chris Trezzo | Chris Trezzo | +| [YARN-4101](https://issues.apache.org/jira/browse/YARN-4101) | RM should print alert messages if Zookeeper and Resourcemanager gets connection issue | Critical | yarn | Yesha Vora | Xuan Gong | +| [YARN-3970](https://issues.apache.org/jira/browse/YARN-3970) | REST api support for Application Priority | Major | webapp | Sunil G | Naganarasimha G R | +| [HDFS-9002](https://issues.apache.org/jira/browse/HDFS-9002) | Move o.a.h.hdfs.net/\*Peer classes to hdfs-client | Major | build | Mingliang Liu | Mingliang Liu | +| [HDFS-9012](https://issues.apache.org/jira/browse/HDFS-9012) | Move o.a.h.hdfs.protocol.datatransfer.PipelineAck class to hadoop-hdfs-client module | Major | build | Mingliang Liu | Mingliang Liu | +| [HDFS-8984](https://issues.apache.org/jira/browse/HDFS-8984) | Move replication queues related methods in FSNamesystem to BlockManager | Major | . | Haohui Mai | Haohui Mai | +| [YARN-2884](https://issues.apache.org/jira/browse/YARN-2884) | Proxying all AM-RM communications | Major | nodemanager, resourcemanager | Carlo Curino | Kishore Chaliparambil | +| [YARN-4136](https://issues.apache.org/jira/browse/YARN-4136) | LinuxContainerExecutor loses info when forwarding ResourceHandlerException | Trivial | nodemanager | Steve Loughran | Bibin A Chundatt | +| [HDFS-9041](https://issues.apache.org/jira/browse/HDFS-9041) | Move entries in META-INF/services/o.a.h.fs.FileSystem to hdfs-client | Major | build | Haohui Mai | Mingliang Liu | +| [HDFS-9010](https://issues.apache.org/jira/browse/HDFS-9010) | Replace NameNode.DEFAULT\_PORT with HdfsClientConfigKeys.DFS\_NAMENODE\_RPC\_PORT\_DEFAULT config key | Major | build | Mingliang Liu | Mingliang Liu | +| [YARN-1651](https://issues.apache.org/jira/browse/YARN-1651) | CapacityScheduler side changes to support increase/decrease container resource. | Major | resourcemanager, scheduler | Wangda Tan | Wangda Tan | +| [YARN-313](https://issues.apache.org/jira/browse/YARN-313) | Add Admin API for supporting node resource configuration in command line | Critical | client, graceful | Junping Du | Inigo Goiri | +| [HDFS-9008](https://issues.apache.org/jira/browse/HDFS-9008) | Balancer#Parameters class could use a builder pattern | Minor | balancer & mover | Chris Trezzo | Chris Trezzo | +| [YARN-3635](https://issues.apache.org/jira/browse/YARN-3635) | Get-queue-mapping should be a common interface of YarnScheduler | Major | scheduler | Wangda Tan | Tan, Wangda | +| [YARN-3717](https://issues.apache.org/jira/browse/YARN-3717) | Expose app/am/queue's node-label-expression to RM web UI / CLI / REST-API | Major | . | Naganarasimha G R | Naganarasimha G R | +| [HDFS-7986](https://issues.apache.org/jira/browse/HDFS-7986) | Allow files / directories to be deleted from the NameNode UI | Major | ui | Ravi Prakash | Ravi Prakash | +| [HDFS-7995](https://issues.apache.org/jira/browse/HDFS-7995) | Implement chmod in the HDFS Web UI | Major | ui | Ravi Prakash | Ravi Prakash | +| [YARN-4034](https://issues.apache.org/jira/browse/YARN-4034) | Render cluster Max Priority in scheduler metrics in RM web UI | Minor | resourcemanager, webapp | Rohith Sharma K S | Rohith Sharma K S | +| [HDFS-9022](https://issues.apache.org/jira/browse/HDFS-9022) | Move NameNode.getAddress() and NameNode.getUri() to hadoop-hdfs-client | Major | hdfs-client | Mingliang Liu | Mingliang Liu | +| [YARN-4171](https://issues.apache.org/jira/browse/YARN-4171) | Resolve findbugs/javac warnings in YARN-1197 branch | Major | api, nodemanager, resourcemanager | Wangda Tan | Wangda Tan | +| [YARN-3212](https://issues.apache.org/jira/browse/YARN-3212) | RMNode State Transition Update with DECOMMISSIONING state | Major | graceful, resourcemanager | Junping Du | Junping Du | +| [HDFS-9101](https://issues.apache.org/jira/browse/HDFS-9101) | Remove deprecated NameNode.getUri() static helper method | Major | . | Mingliang Liu | Mingliang Liu | +| [HDFS-9004](https://issues.apache.org/jira/browse/HDFS-9004) | Add upgrade domain to DatanodeInfo | Major | . | Ming Ma | Ming Ma | +| [HDFS-9111](https://issues.apache.org/jira/browse/HDFS-9111) | Move hdfs-client protobuf convert methods from PBHelper to PBHelperClient | Major | . | Mingliang Liu | Mingliang Liu | +| [HDFS-9039](https://issues.apache.org/jira/browse/HDFS-9039) | Separate client and server side methods of o.a.h.hdfs.NameNodeProxies | Major | . | Mingliang Liu | Mingliang Liu | +| [HDFS-8733](https://issues.apache.org/jira/browse/HDFS-8733) | Keep server related definition in hdfs.proto on server side | Major | . | Yi Liu | Mingliang Liu | +| [HDFS-9131](https://issues.apache.org/jira/browse/HDFS-9131) | Move config keys used by hdfs-client to HdfsClientConfigKeys | Major | . | Mingliang Liu | Mingliang Liu | +| [HDFS-7529](https://issues.apache.org/jira/browse/HDFS-7529) | Consolidate encryption zone related implementation into a single class | Major | . | Haohui Mai | Rakesh R | +| [HDFS-9134](https://issues.apache.org/jira/browse/HDFS-9134) | Move LEASE\_{SOFTLIMIT,HARDLIMIT}\_PERIOD constants from HdfsServerConstants to HdfsConstants | Major | . | Mingliang Liu | Mingliang Liu | +| [HADOOP-11918](https://issues.apache.org/jira/browse/HADOOP-11918) | Listing an empty s3a root directory throws FileNotFound. | Minor | fs/s3 | Lei (Eddy) Xu | Lei (Eddy) Xu | +| [HDFS-8053](https://issues.apache.org/jira/browse/HDFS-8053) | Move DFSIn/OutputStream and related classes to hadoop-hdfs-client | Major | build | Haohui Mai | Mingliang Liu | +| [HDFS-8740](https://issues.apache.org/jira/browse/HDFS-8740) | Move DistributedFileSystem to hadoop-hdfs-client | Major | build | Yi Liu | Mingliang Liu | +| [YARN-4141](https://issues.apache.org/jira/browse/YARN-4141) | Runtime Application Priority change should not throw exception for applications at finishing states | Major | resourcemanager | Sunil G | Sunil G | +| [HDFS-9165](https://issues.apache.org/jira/browse/HDFS-9165) | Move entries in META-INF/services/o.a.h.fs.FileSystem to hdfs-client | Major | build | Haohui Mai | Mingliang Liu | +| [HDFS-9166](https://issues.apache.org/jira/browse/HDFS-9166) | Move hftp / hsftp filesystem to hfds-client | Major | build | Haohui Mai | Mingliang Liu | +| [HDFS-8971](https://issues.apache.org/jira/browse/HDFS-8971) | Remove guards when calling LOG.debug() and LOG.trace() in client package | Major | build | Mingliang Liu | Mingliang Liu | +| [HDFS-9015](https://issues.apache.org/jira/browse/HDFS-9015) | Refactor TestReplicationPolicy to test different block placement policies | Major | . | Ming Ma | Ming Ma | +| [YARN-1897](https://issues.apache.org/jira/browse/YARN-1897) | CLI and core support for signal container functionality | Major | api | Ming Ma | Ming Ma | +| [HDFS-9158](https://issues.apache.org/jira/browse/HDFS-9158) | [OEV-Doc] : Document does not mention about "-f" and "-r" options | Major | . | nijel | nijel | +| [HDFS-9155](https://issues.apache.org/jira/browse/HDFS-9155) | OEV should treat .XML files as XML even when the file name extension is uppercase | Major | . | nijel | nijel | +| [YARN-4215](https://issues.apache.org/jira/browse/YARN-4215) | RMNodeLabels Manager Need to verify and replace node labels for the only modified Node Label Mappings in the request | Major | resourcemanager | Naganarasimha G R | Naganarasimha G R | +| [HDFS-9170](https://issues.apache.org/jira/browse/HDFS-9170) | Move libhdfs / fuse-dfs / libwebhdfs to hdfs-client | Major | . | Haohui Mai | Haohui Mai | +| [HDFS-9159](https://issues.apache.org/jira/browse/HDFS-9159) | [OIV] : return value of the command is not correct if invalid value specified in "-p (processor)" option | Major | . | nijel | nijel | +| [YARN-4140](https://issues.apache.org/jira/browse/YARN-4140) | RM container allocation delayed incase of app submitted to Nodelabel partition | Major | scheduler | Bibin A Chundatt | Bibin A Chundatt | +| [YARN-3964](https://issues.apache.org/jira/browse/YARN-3964) | Support NodeLabelsProvider at Resource Manager side | Major | . | Dian Fu | Dian Fu | +| [YARN-4230](https://issues.apache.org/jira/browse/YARN-4230) | Increasing container resource while there is no headroom left will cause ResourceManager to crash | Critical | resourcemanager | MENG DING | MENG DING | +| [HDFS-9006](https://issues.apache.org/jira/browse/HDFS-9006) | Provide BlockPlacementPolicy that supports upgrade domain | Major | . | Ming Ma | Ming Ma | +| [HDFS-9160](https://issues.apache.org/jira/browse/HDFS-9160) | [OIV-Doc] : Missing details of "delimited" for processor options | Major | . | nijel | nijel | +| [HDFS-9167](https://issues.apache.org/jira/browse/HDFS-9167) | Update pom.xml in other modules to depend on hdfs-client instead of hdfs | Major | build | Haohui Mai | Mingliang Liu | +| [YARN-4255](https://issues.apache.org/jira/browse/YARN-4255) | container-executor does not clean up docker operation command files. | Minor | . | Sidharta Seethana | Sidharta Seethana | +| [HDFS-9223](https://issues.apache.org/jira/browse/HDFS-9223) | Code cleanup for DatanodeDescriptor and HeartbeatManager | Minor | namenode | Jing Zhao | Jing Zhao | +| [YARN-4258](https://issues.apache.org/jira/browse/YARN-4258) | Add support for controlling capabilities for docker containers | Major | yarn | Sidharta Seethana | Sidharta Seethana | +| [HDFS-9157](https://issues.apache.org/jira/browse/HDFS-9157) | [OEV and OIV] : Unnecessary parsing for mandatory arguements if "-h" option is specified as the only option | Major | . | nijel | nijel | +| [HADOOP-12475](https://issues.apache.org/jira/browse/HADOOP-12475) | Replace guava Cache with ConcurrentHashMap for caching Connection in ipc Client | Major | conf, io, ipc | Walter Su | Walter Su | +| [YARN-4162](https://issues.apache.org/jira/browse/YARN-4162) | CapacityScheduler: Add resource usage by partition and queue capacity by partition to REST API | Major | api, client, resourcemanager | Naganarasimha G R | Naganarasimha G R | +| [YARN-4170](https://issues.apache.org/jira/browse/YARN-4170) | AM need to be notified with priority in AllocateResponse | Major | resourcemanager | Sunil G | Sunil G | +| [YARN-2556](https://issues.apache.org/jira/browse/YARN-2556) | Tool to measure the performance of the timeline server | Major | timelineserver | Jonathan Eagles | Chang Li | +| [YARN-4262](https://issues.apache.org/jira/browse/YARN-4262) | Allow whitelisted users to run privileged docker containers. | Major | yarn | Sidharta Seethana | Sidharta Seethana | +| [YARN-4267](https://issues.apache.org/jira/browse/YARN-4267) | Add additional logging to container launch implementations in container-executor | Major | yarn | Sidharta Seethana | Sidharta Seethana | +| [YARN-3985](https://issues.apache.org/jira/browse/YARN-3985) | Make ReservationSystem persist state using RMStateStore reservation APIs | Major | resourcemanager | Anubhav Dhoot | Anubhav Dhoot | +| [YARN-2513](https://issues.apache.org/jira/browse/YARN-2513) | Host framework UIs in YARN for use with the ATS | Major | timelineserver | Jonathan Eagles | Jonathan Eagles | +| [YARN-3739](https://issues.apache.org/jira/browse/YARN-3739) | Add reservation system recovery to RM recovery process | Major | capacityscheduler, fairscheduler, resourcemanager | Subru Krishnan | Subru Krishnan | +| [YARN-4243](https://issues.apache.org/jira/browse/YARN-4243) | Add retry on establishing Zookeeper conenction in EmbeddedElectorService#serviceInit | Major | resourcemanager | Xuan Gong | Xuan Gong | +| [YARN-3738](https://issues.apache.org/jira/browse/YARN-3738) | Add support for recovery of reserved apps running under dynamic queues | Major | capacityscheduler, resourcemanager | Subru Krishnan | Subru Krishnan | +| [YARN-3724](https://issues.apache.org/jira/browse/YARN-3724) | Use POSIX nftw(3) instead of fts(3) | Major | . | Malcolm Kavalsky | Alan Burlison | +| [YARN-2729](https://issues.apache.org/jira/browse/YARN-2729) | Support script based NodeLabelsProvider Interface in Distributed Node Label Configuration Setup | Major | nodemanager | Naganarasimha G R | Naganarasimha G R | +| [HDFS-9304](https://issues.apache.org/jira/browse/HDFS-9304) | Add HdfsClientConfigKeys class to TestHdfsConfigFields#configurationClasses | Major | build | Mingliang Liu | Mingliang Liu | +| [YARN-3216](https://issues.apache.org/jira/browse/YARN-3216) | Max-AM-Resource-Percentage should respect node labels | Critical | resourcemanager | Wangda Tan | Sunil G | +| [HADOOP-12457](https://issues.apache.org/jira/browse/HADOOP-12457) | [JDK8] Fix a failure of compiling common by javadoc | Major | . | Tsuyoshi Ozawa | Akira Ajisaka | +| [HDFS-9168](https://issues.apache.org/jira/browse/HDFS-9168) | Move client side unit test to hadoop-hdfs-client | Major | build | Haohui Mai | Haohui Mai | +| [HDFS-9343](https://issues.apache.org/jira/browse/HDFS-9343) | Empty caller context considered invalid | Major | . | Mingliang Liu | Mingliang Liu | +| [HDFS-9362](https://issues.apache.org/jira/browse/HDFS-9362) | TestAuditLogger#testAuditLoggerWithCallContext assumes Unix line endings, fails on Windows. | Minor | test | Chris Nauroth | Chris Nauroth | +| [HDFS-9007](https://issues.apache.org/jira/browse/HDFS-9007) | Fix HDFS Balancer to honor upgrade domain policy | Major | . | Ming Ma | Ming Ma | +| [HDFS-9379](https://issues.apache.org/jira/browse/HDFS-9379) | Make NNThroughputBenchmark$BlockReportStats support more than 10 datanodes | Major | test | Mingliang Liu | Mingliang Liu | +| [YARN-1510](https://issues.apache.org/jira/browse/YARN-1510) | Make NMClient support change container resources | Major | nodemanager | Wangda Tan (No longer used) | MENG DING | +| [YARN-4345](https://issues.apache.org/jira/browse/YARN-4345) | yarn rmadmin -updateNodeResource doesn't work | Critical | graceful, resourcemanager | Sushmitha Sreenivasan | Junping Du | +| [YARN-1509](https://issues.apache.org/jira/browse/YARN-1509) | Make AMRMClient support send increase container request and get increased/decreased containers | Major | resourcemanager | Wangda Tan (No longer used) | MENG DING | +| [HDFS-9387](https://issues.apache.org/jira/browse/HDFS-9387) | Fix namenodeUri parameter parsing in NNThroughputBenchmark | Major | test | Mingliang Liu | Mingliang Liu | +| [HDFS-9421](https://issues.apache.org/jira/browse/HDFS-9421) | NNThroughputBenchmark replication test NPE with -namenode option | Major | benchmarks | Xiaoyu Yao | Mingliang Liu | +| [YARN-4184](https://issues.apache.org/jira/browse/YARN-4184) | Remove update reservation state api from state store as its not used by ReservationSystem | Major | capacityscheduler, fairscheduler, resourcemanager | Anubhav Dhoot | Sean Po | +| [HADOOP-12582](https://issues.apache.org/jira/browse/HADOOP-12582) | Using BytesWritable's getLength() and getBytes() instead of get() and getSize() | Major | . | Tsuyoshi Ozawa | Akira Ajisaka | +| [YARN-3454](https://issues.apache.org/jira/browse/YARN-3454) | Add efficient merge operation to RLESparseResourceAllocation | Major | resourcemanager | Carlo Curino | Carlo Curino | +| [HDFS-7796](https://issues.apache.org/jira/browse/HDFS-7796) | Include X-editable for slick contenteditable fields in the web UI | Major | ui | Ravi Prakash | Ravi Prakash | +| [YARN-3980](https://issues.apache.org/jira/browse/YARN-3980) | Plumb resource-utilization info in node heartbeat through to the scheduler | Major | resourcemanager, scheduler | Karthik Kambatla | Inigo Goiri | +| [HADOOP-11954](https://issues.apache.org/jira/browse/HADOOP-11954) | Solaris does not support RLIMIT\_MEMLOCK as in Linux | Major | . | Malcolm Kavalsky | Alan Burlison | +| [YARN-4384](https://issues.apache.org/jira/browse/YARN-4384) | updateNodeResource CLI should not accept negative values for resource | Major | graceful, resourcemanager | Sushmitha Sreenivasan | Junping Du | +| [HDFS-9438](https://issues.apache.org/jira/browse/HDFS-9438) | TestPipelinesFailover assumes Linux ifconfig | Minor | test | Alan Burlison | John Zhuge | +| [YARN-4292](https://issues.apache.org/jira/browse/YARN-4292) | ResourceUtilization should be a part of NodeInfo REST API | Major | . | Wangda Tan | Sunil G | +| [HDFS-9436](https://issues.apache.org/jira/browse/HDFS-9436) | Make NNThroughputBenchmark$BlockReportStats run with 10 datanodes by default | Minor | test | Mingliang Liu | Mingliang Liu | +| [HDFS-9484](https://issues.apache.org/jira/browse/HDFS-9484) | NNThroughputBenchmark$BlockReportStats should not send empty block reports | Major | test | Mingliang Liu | Mingliang Liu | +| [YARN-4405](https://issues.apache.org/jira/browse/YARN-4405) | Support node label store in non-appendable file system | Major | api, client, resourcemanager | Wangda Tan | Wangda Tan | +| [HDFS-9214](https://issues.apache.org/jira/browse/HDFS-9214) | Support reconfiguring dfs.datanode.balance.max.concurrent.moves without DN restart | Major | datanode | Xiaobing Zhou | Xiaobing Zhou | +| [YARN-4248](https://issues.apache.org/jira/browse/YARN-4248) | REST API for submit/update/delete Reservations | Major | resourcemanager | Carlo Curino | Carlo Curino | +| [YARN-4358](https://issues.apache.org/jira/browse/YARN-4358) | Improve relationship between SharingPolicy and ReservationAgent | Major | capacityscheduler, fairscheduler, resourcemanager | Carlo Curino | Carlo Curino | +| [YARN-3946](https://issues.apache.org/jira/browse/YARN-3946) | Update exact reason as to why a submitted app is in ACCEPTED state to app's diagnostic message | Major | capacity scheduler, resourcemanager | Sumit Nigam | Naganarasimha G R | +| [YARN-4309](https://issues.apache.org/jira/browse/YARN-4309) | Add container launch related debug information to container logs when a container fails | Major | nodemanager | Varun Vasudev | Varun Vasudev | +| [YARN-4293](https://issues.apache.org/jira/browse/YARN-4293) | ResourceUtilization should be a part of yarn node CLI | Major | . | Wangda Tan | Sunil G | +| [YARN-4416](https://issues.apache.org/jira/browse/YARN-4416) | Deadlock due to synchronised get Methods in AbstractCSQueue | Minor | capacity scheduler, resourcemanager | Naganarasimha G R | Naganarasimha G R | +| [YARN-3226](https://issues.apache.org/jira/browse/YARN-3226) | UI changes for decommissioning node | Major | graceful | Junping Du | Sunil G | +| [YARN-4164](https://issues.apache.org/jira/browse/YARN-4164) | Retrospect update ApplicationPriority API return type | Major | resourcemanager | Rohith Sharma K S | Rohith Sharma K S | +| [YARN-4234](https://issues.apache.org/jira/browse/YARN-4234) | New put APIs in TimelineClient for ats v1.5 | Major | timelineserver | Xuan Gong | Xuan Gong | +| [YARN-4098](https://issues.apache.org/jira/browse/YARN-4098) | Document ApplicationPriority feature | Major | resourcemanager | Rohith Sharma K S | Rohith Sharma K S | +| [HDFS-7779](https://issues.apache.org/jira/browse/HDFS-7779) | Support changing ownership, group and replication in HDFS Web UI | Major | ui | Ravi Prakash | Ravi Prakash | +| [YARN-2902](https://issues.apache.org/jira/browse/YARN-2902) | Killing a container that is localizing can orphan resources in the DOWNLOADING state | Major | nodemanager | Jason Lowe | Varun Saxena | +| [YARN-4393](https://issues.apache.org/jira/browse/YARN-4393) | TestResourceLocalizationService#testFailedDirsResourceRelease fails intermittently | Major | test | Varun Saxena | Varun Saxena | +| [YARN-4479](https://issues.apache.org/jira/browse/YARN-4479) | Retrospect app-priority in pendingOrderingPolicy during recovering applications | Major | api, resourcemanager | Rohith Sharma K S | Rohith Sharma K S | +| [HDFS-9621](https://issues.apache.org/jira/browse/HDFS-9621) | getListing wrongly associates Erasure Coding policy to pre-existing replicated files under an EC directory | Critical | erasure-coding | Sushmitha Sreenivasan | Jing Zhao | +| [YARN-4537](https://issues.apache.org/jira/browse/YARN-4537) | Pull out priority comparison from fifocomparator and use compound comparator for FifoOrdering policy | Major | capacity scheduler | Rohith Sharma K S | Rohith Sharma K S | +| [HADOOP-11262](https://issues.apache.org/jira/browse/HADOOP-11262) | Enable YARN to use S3A | Major | fs/s3 | Thomas Demoor | Pieter Reuse | +| [YARN-4265](https://issues.apache.org/jira/browse/YARN-4265) | Provide new timeline plugin storage to support fine-grained entity caching | Major | timelineserver | Li Lu | Li Lu | +| [YARN-4304](https://issues.apache.org/jira/browse/YARN-4304) | AM max resource configuration per partition to be displayed/updated correctly in UI and in various partition related metrics | Major | webapp | Sunil G | Sunil G | +| [YARN-4557](https://issues.apache.org/jira/browse/YARN-4557) | Fix improper Queues sorting in PartitionedQueueComparator when accessible-node-labels=\* | Major | resourcemanager | Naganarasimha G R | Naganarasimha G R | +| [HDFS-9601](https://issues.apache.org/jira/browse/HDFS-9601) | NNThroughputBenchmark.BlockReportStats should handle NotReplicatedYetException on adding block | Major | test | Masatake Iwasaki | Masatake Iwasaki | +| [YARN-4614](https://issues.apache.org/jira/browse/YARN-4614) | TestApplicationPriority#testApplicationPriorityAllocationWithChangeInPriority fails occasionally | Major | test | Jason Lowe | Sunil G | +| [HDFS-9672](https://issues.apache.org/jira/browse/HDFS-9672) | o.a.h.hdfs.TestLeaseRecovery2 fails intermittently | Major | test | Mingliang Liu | Mingliang Liu | +| [YARN-4573](https://issues.apache.org/jira/browse/YARN-4573) | TestRMAppTransitions.testAppRunningKill and testAppKilledKilled fail on trunk | Major | resourcemanager, test | Takashi Ohnishi | Takashi Ohnishi | +| [YARN-4643](https://issues.apache.org/jira/browse/YARN-4643) | Container recovery is broken with delegating container runtime | Critical | yarn | Sidharta Seethana | Sidharta Seethana | +| [YARN-4219](https://issues.apache.org/jira/browse/YARN-4219) | New levelDB cache storage for timeline v1.5 | Major | . | Li Lu | Li Lu | +| [YARN-4543](https://issues.apache.org/jira/browse/YARN-4543) | TestNodeStatusUpdater.testStopReentrant fails + JUnit misusage | Minor | nodemanager | Akihiro Suda | Akihiro Suda | +| [YARN-4340](https://issues.apache.org/jira/browse/YARN-4340) | Add "list" API to reservation system | Major | capacityscheduler, fairscheduler, resourcemanager | Carlo Curino | Sean Po | +| [YARN-4100](https://issues.apache.org/jira/browse/YARN-4100) | Add Documentation for Distributed and Delegated-Centralized Node Labels feature | Major | api, client, resourcemanager | Naganarasimha G R | Naganarasimha G R | +| [HDFS-9503](https://issues.apache.org/jira/browse/HDFS-9503) | Replace -namenode option with -fs for NNThroughputBenchmark | Major | test | Konstantin Shvachko | Mingliang Liu | +| [HADOOP-12292](https://issues.apache.org/jira/browse/HADOOP-12292) | Make use of DeleteObjects optional | Major | fs/s3 | Thomas Demoor | Thomas Demoor | +| [YARN-4667](https://issues.apache.org/jira/browse/YARN-4667) | RM Admin CLI for refreshNodesResources throws NPE when nothing is configured | Critical | client | Naganarasimha G R | Naganarasimha G R | +| [HADOOP-12752](https://issues.apache.org/jira/browse/HADOOP-12752) | Improve diagnostics/use of envvar/sysprop credential propagation | Minor | security | Steve Loughran | Steve Loughran | +| [YARN-4138](https://issues.apache.org/jira/browse/YARN-4138) | Roll back container resource allocation after resource increase token expires | Major | api, nodemanager, resourcemanager | MENG DING | MENG DING | +| [YARN-2575](https://issues.apache.org/jira/browse/YARN-2575) | Create separate ACLs for Reservation create/update/delete/list ops | Major | capacityscheduler, fairscheduler, resourcemanager | Subru Krishnan | Sean Po | +| [HADOOP-11613](https://issues.apache.org/jira/browse/HADOOP-11613) | Remove commons-httpclient dependency from hadoop-azure | Major | . | Akira Ajisaka | Masatake Iwasaki | +| [HDFS-9084](https://issues.apache.org/jira/browse/HDFS-9084) | Pagination, sorting and filtering of files/directories in the HDFS Web UI | Major | ui | Ravi Prakash | Ravi Prakash | +| [YARN-3223](https://issues.apache.org/jira/browse/YARN-3223) | Resource update during NM graceful decommission | Major | graceful, nodemanager, resourcemanager | Junping Du | Brook Zhou | +| [YARN-4680](https://issues.apache.org/jira/browse/YARN-4680) | TimerTasks leak in ATS V1.5 Writer | Major | timelineserver | Xuan Gong | Xuan Gong | +| [HADOOP-12711](https://issues.apache.org/jira/browse/HADOOP-12711) | Remove dependency on commons-httpclient for ServletUtil | Major | . | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [YARN-4566](https://issues.apache.org/jira/browse/YARN-4566) | TestMiniYarnClusterNodeUtilization sometimes fails on trunk | Major | test | Takashi Ohnishi | Takashi Ohnishi | +| [HADOOP-12813](https://issues.apache.org/jira/browse/HADOOP-12813) | Migrate TestRPC and related codes to rebase on ProtobufRpcEngine | Major | . | Kai Zheng | Kai Zheng | +| [YARN-4749](https://issues.apache.org/jira/browse/YARN-4749) | Generalize config file handling in container-executor | Major | nodemanager | Sidharta Seethana | Sidharta Seethana | +| [YARN-4696](https://issues.apache.org/jira/browse/YARN-4696) | Improving EntityGroupFSTimelineStore on exception handling, test setup, and concurrency | Major | timelineserver | Steve Loughran | Steve Loughran | +| [MAPREDUCE-6520](https://issues.apache.org/jira/browse/MAPREDUCE-6520) | Migrate MR Client test cases part 1 | Trivial | test | Dustin Cote | Dustin Cote | +| [YARN-4545](https://issues.apache.org/jira/browse/YARN-4545) | Allow YARN distributed shell to use ATS v1.5 APIs | Major | timelineserver | Li Lu | Li Lu | +| [YARN-4817](https://issues.apache.org/jira/browse/YARN-4817) | Change Log Level to DEBUG for putDomain call in ATS 1.5 | Trivial | timelineserver | Xuan Gong | Xuan Gong | +| [YARN-4108](https://issues.apache.org/jira/browse/YARN-4108) | CapacityScheduler: Improve preemption to only kill containers that would satisfy the incoming request | Major | capacity scheduler | Wangda Tan | Wangda Tan | +| [HADOOP-12819](https://issues.apache.org/jira/browse/HADOOP-12819) | Migrate TestSaslRPC and related codes to rebase on ProtobufRpcEngine | Major | . | Kai Zheng | Kai Zheng | +| [YARN-4815](https://issues.apache.org/jira/browse/YARN-4815) | ATS 1.5 timelineclient impl try to create attempt directory for every event call | Major | timelineserver | Xuan Gong | Xuan Gong | +| [YARN-4814](https://issues.apache.org/jira/browse/YARN-4814) | ATS 1.5 timelineclient impl call flush after every event write | Major | timelineserver | Xuan Gong | Xuan Gong | +| [YARN-998](https://issues.apache.org/jira/browse/YARN-998) | Keep NM resource updated through dynamic resource config for RM/NM restart | Major | graceful, nodemanager, scheduler | Junping Du | Junping Du | +| [MAPREDUCE-6543](https://issues.apache.org/jira/browse/MAPREDUCE-6543) | Migrate MR Client test cases part 2 | Trivial | test | Dustin Cote | Dustin Cote | +| [YARN-4822](https://issues.apache.org/jira/browse/YARN-4822) | Refactor existing Preemption Policy of CS for easier adding new approach to select preemption candidates | Major | . | Wangda Tan | Wangda Tan | +| [YARN-4634](https://issues.apache.org/jira/browse/YARN-4634) | Scheduler UI/Metrics need to consider cases like non-queue label mappings | Major | . | Sunil G | Sunil G | +| [HADOOP-12169](https://issues.apache.org/jira/browse/HADOOP-12169) | ListStatus on empty dir in S3A lists itself instead of returning an empty list | Major | fs/s3 | Pieter Reuse | Pieter Reuse | +| [HDFS-10186](https://issues.apache.org/jira/browse/HDFS-10186) | DirectoryScanner: Improve logs by adding full path of both actual and expected block directories | Minor | datanode | Rakesh R | Rakesh R | +| [YARN-4826](https://issues.apache.org/jira/browse/YARN-4826) | Document configuration of ReservationSystem for CapacityScheduler | Minor | capacity scheduler | Subru Krishnan | Subru Krishnan | +| [HADOOP-12444](https://issues.apache.org/jira/browse/HADOOP-12444) | Support lazy seek in S3AInputStream | Major | fs/s3 | Rajesh Balamohan | Rajesh Balamohan | +| [YARN-4928](https://issues.apache.org/jira/browse/YARN-4928) | Some yarn.server.timeline.\* tests fail on Windows attempting to use a test root path containing a colon | Minor | test | Gergely Novák | Gergely Novák | +| [YARN-4168](https://issues.apache.org/jira/browse/YARN-4168) | Test TestLogAggregationService.testLocalFileDeletionOnDiskFull failing | Critical | test | Steve Loughran | Takashi Ohnishi | +| [HADOOP-12973](https://issues.apache.org/jira/browse/HADOOP-12973) | make DU pluggable | Major | . | Elliott Clark | Elliott Clark | +| [YARN-4886](https://issues.apache.org/jira/browse/YARN-4886) | Add HDFS caller context for EntityGroupFSTimelineStore | Major | timelineserver | Li Lu | Li Lu | +| [HDFS-10281](https://issues.apache.org/jira/browse/HDFS-10281) | o.a.h.hdfs.server.namenode.ha.TestPendingCorruptDnMessages fails intermittently | Major | test | Mingliang Liu | Mingliang Liu | +| [YARN-4909](https://issues.apache.org/jira/browse/YARN-4909) | Fix intermittent failures of TestRMWebServices And TestRMWithCSRFFilter | Blocker | . | Brahma Reddy Battula | Bibin A Chundatt | +| [YARN-4468](https://issues.apache.org/jira/browse/YARN-4468) | Document the general ReservationSystem functionality, and the REST API | Major | capacityscheduler, fairscheduler, resourcemanager | Carlo Curino | Carlo Curino | +| [HADOOP-13011](https://issues.apache.org/jira/browse/HADOOP-13011) | Clearly Document the Password Details for Keystore-based Credential Providers | Major | documentation | Larry McCay | Larry McCay | +| [YARN-3215](https://issues.apache.org/jira/browse/YARN-3215) | Respect labels in CapacityScheduler when computing headroom | Major | capacityscheduler | Wangda Tan | Naganarasimha G R | +| [HDFS-10224](https://issues.apache.org/jira/browse/HDFS-10224) | Implement asynchronous rename for DistributedFileSystem | Major | fs, hdfs-client | Xiaobing Zhou | Xiaobing Zhou | +| [YARN-4956](https://issues.apache.org/jira/browse/YARN-4956) | findbug issue on LevelDBCacheTimelineStore | Major | timelineserver | Xuan Gong | Zhiyuan Yang | +| [YARN-4851](https://issues.apache.org/jira/browse/YARN-4851) | Metric improvements for ATS v1.5 storage components | Major | timelineserver | Li Lu | Li Lu | +| [HADOOP-12749](https://issues.apache.org/jira/browse/HADOOP-12749) | Create a threadpoolexecutor that overrides afterExecute to log uncaught exceptions/errors | Major | . | Sidharta Seethana | Sidharta Seethana | +| [HDFS-10346](https://issues.apache.org/jira/browse/HDFS-10346) | Implement asynchronous setPermission/setOwner for DistributedFileSystem | Major | hdfs, hdfs-client | Xiaobing Zhou | Xiaobing Zhou | +| [HADOOP-13122](https://issues.apache.org/jira/browse/HADOOP-13122) | Customize User-Agent header sent in HTTP requests by S3A. | Minor | fs/s3 | Chris Nauroth | Chris Nauroth | +| [HADOOP-12844](https://issues.apache.org/jira/browse/HADOOP-12844) | Recover when S3A fails on IOException in read() | Major | fs/s3 | Pieter Reuse | Pieter Reuse | +| [HADOOP-13028](https://issues.apache.org/jira/browse/HADOOP-13028) | add low level counter metrics for S3A; use in read performance tests | Major | fs/s3, metrics | Steve Loughran | Steve Loughran | +| [HADOOP-13113](https://issues.apache.org/jira/browse/HADOOP-13113) | Enable parallel test execution for hadoop-aws. | Minor | test | Chris Nauroth | Chris Nauroth | +| [HADOOP-13158](https://issues.apache.org/jira/browse/HADOOP-13158) | S3AFileSystem#toString might throw NullPointerException due to null cannedACL. | Minor | fs/s3 | Chris Nauroth | Chris Nauroth | +| [YARN-4832](https://issues.apache.org/jira/browse/YARN-4832) | NM side resource value should get updated if change applied in RM side | Critical | nodemanager, resourcemanager | Junping Du | Junping Du | +| [HADOOP-13140](https://issues.apache.org/jira/browse/HADOOP-13140) | FileSystem#initialize must not attempt to create StorageStatistics objects with null or empty schemes | Major | fs | Brahma Reddy Battula | Mingliang Liu | +| [HADOOP-13130](https://issues.apache.org/jira/browse/HADOOP-13130) | s3a failures can surface as RTEs, not IOEs | Major | fs/s3 | Steve Loughran | Steve Loughran | +| [YARN-3362](https://issues.apache.org/jira/browse/YARN-3362) | Add node label usage in RM CapacityScheduler web UI | Major | capacityscheduler, resourcemanager, webapp | Wangda Tan | Naganarasimha G R | +| [HDFS-10390](https://issues.apache.org/jira/browse/HDFS-10390) | Implement asynchronous setAcl/getAclStatus for DistributedFileSystem | Major | fs | Xiaobing Zhou | Xiaobing Zhou | +| [YARN-857](https://issues.apache.org/jira/browse/YARN-857) | Localization failures should be available in container diagnostics | Critical | . | Hitesh Shah | Vinod Kumar Vavilapalli | +| [HDFS-8057](https://issues.apache.org/jira/browse/HDFS-8057) | Move BlockReader implementation to the client implementation package | Major | hdfs-client | Tsz Wo Nicholas Sze | Takanobu Asanuma | +| [YARN-4957](https://issues.apache.org/jira/browse/YARN-4957) | Add getNewReservation in ApplicationClientProtocol | Major | applications, client, resourcemanager | Subru Krishnan | Sean Po | +| [HDFS-10431](https://issues.apache.org/jira/browse/HDFS-10431) | Refactor and speedup TestAsyncDFSRename | Minor | test | Xiaobing Zhou | Xiaobing Zhou | +| [YARN-4987](https://issues.apache.org/jira/browse/YARN-4987) | Read cache concurrency issue between read and evict in EntityGroupFS timeline store | Critical | . | Li Lu | Li Lu | +| [HDFS-10430](https://issues.apache.org/jira/browse/HDFS-10430) | Reuse FileSystem#access in TestAsyncDFS | Major | hdfs | Xiaobing Zhou | Xiaobing Zhou | +| [HADOOP-13162](https://issues.apache.org/jira/browse/HADOOP-13162) | Consider reducing number of getFileStatus calls in S3AFileSystem.mkdirs | Minor | fs/s3 | Rajesh Balamohan | Rajesh Balamohan | +| [HADOOP-13131](https://issues.apache.org/jira/browse/HADOOP-13131) | Add tests to verify that S3A supports SSE-S3 encryption | Minor | fs/s3 | Steve Loughran | Steve Loughran | +| [HADOOP-13171](https://issues.apache.org/jira/browse/HADOOP-13171) | Add StorageStatistics to S3A; instrument some more operations | Minor | fs/s3 | Steve Loughran | Steve Loughran | +| [YARN-1815](https://issues.apache.org/jira/browse/YARN-1815) | Work preserving recovery of Unmanged AMs | Critical | resourcemanager | Karthik Kambatla | Subru Krishnan | +| [YARN-5165](https://issues.apache.org/jira/browse/YARN-5165) | Fix NoOvercommitPolicy to take advantage of RLE representation of plan | Major | capacityscheduler, fairscheduler, resourcemanager | Carlo Curino | Carlo Curino | +| [YARN-5185](https://issues.apache.org/jira/browse/YARN-5185) | StageAllocaterGreedyRLE: Fix NPE in corner case | Major | capacityscheduler, fairscheduler, resourcemanager | Carlo Curino | Carlo Curino | +| [YARN-4525](https://issues.apache.org/jira/browse/YARN-4525) | Fix bug in RLESparseResourceAllocation.getRangeOverlapping(...) | Major | . | Ishai Menache | Ishai Menache | +| [HADOOP-13237](https://issues.apache.org/jira/browse/HADOOP-13237) | s3a initialization against public bucket fails if caller lacks any credentials | Minor | fs/s3 | Steve Loughran | Chris Nauroth | +| [YARN-5199](https://issues.apache.org/jira/browse/YARN-5199) | Close LogReader in in AHSWebServices#getStreamingOutput and FileInputStream in NMWebServices#getLogs | Major | . | Xuan Gong | Xuan Gong | +| [YARN-3426](https://issues.apache.org/jira/browse/YARN-3426) | Add jdiff support to YARN | Blocker | . | Li Lu | Li Lu | +| [YARN-1942](https://issues.apache.org/jira/browse/YARN-1942) | Deprecate toString/fromString methods from ConverterUtils and move them to records classes like ContainerId/ApplicationId, etc. | Critical | api | Thomas Graves | Wangda Tan | +| [HADOOP-13241](https://issues.apache.org/jira/browse/HADOOP-13241) | document s3a better | Minor | documentation, fs/s3 | Steve Loughran | Steve Loughran | +| [HADOOP-13284](https://issues.apache.org/jira/browse/HADOOP-13284) | FileSystemStorageStatistics must not attempt to read non-existent rack-aware read stats in branch-2.8 | Major | fs | Mingliang Liu | Mingliang Liu | +| [HADOOP-12975](https://issues.apache.org/jira/browse/HADOOP-12975) | Add jitter to CachingGetSpaceUsed's thread | Major | . | Elliott Clark | Elliott Clark | +| [HADOOP-13280](https://issues.apache.org/jira/browse/HADOOP-13280) | FileSystemStorageStatistics#getLong(“readOps“) should return readOps + largeReadOps | Major | fs | Mingliang Liu | Mingliang Liu | +| [HADOOP-13288](https://issues.apache.org/jira/browse/HADOOP-13288) | Guard null stats key in FileSystemStorageStatistics | Major | fs | Mingliang Liu | Mingliang Liu | +| [HADOOP-13291](https://issues.apache.org/jira/browse/HADOOP-13291) | Probing stats in DFSOpsCountStatistics/S3AStorageStatistics should be correctly implemented | Major | fs | Mingliang Liu | Mingliang Liu | +| [HDFS-10538](https://issues.apache.org/jira/browse/HDFS-10538) | Remove AsyncDistributedFileSystem API | Major | hdfs-client | Xiaobing Zhou | Xiaobing Zhou | +| [HADOOP-13203](https://issues.apache.org/jira/browse/HADOOP-13203) | S3A: Support fadvise "random" mode for high performance readPositioned() reads | Major | fs/s3 | Rajesh Balamohan | Rajesh Balamohan | +| [HADOOP-12229](https://issues.apache.org/jira/browse/HADOOP-12229) | Fix inconsistent subsection titles in filesystem.md | Minor | documentation | Masatake Iwasaki | Masatake Iwasaki | +| [HADOOP-12242](https://issues.apache.org/jira/browse/HADOOP-12242) | Add in-page TOC to filesystem specification pages | Minor | documentation | Masatake Iwasaki | Masatake Iwasaki | +| [HADOOP-13305](https://issues.apache.org/jira/browse/HADOOP-13305) | Define common statistics names across schemes | Major | fs | Mingliang Liu | Mingliang Liu | +| [HADOOP-13283](https://issues.apache.org/jira/browse/HADOOP-13283) | Support reset operation for new global storage statistics and per FS storage stats | Major | fs | Mingliang Liu | Mingliang Liu | +| [YARN-5080](https://issues.apache.org/jira/browse/YARN-5080) | Cannot obtain logs using YARN CLI -am for either KILLED or RUNNING AM | Critical | yarn | Sumana Sathish | Xuan Gong | +| [HADOOP-13366](https://issues.apache.org/jira/browse/HADOOP-13366) | Fix dead link in o.a.h.fs.CommonConfigurationKeysPublic javadoc | Minor | documentation | Rakesh R | Rakesh R | +| [YARN-4484](https://issues.apache.org/jira/browse/YARN-4484) | Available Resource calculation for a queue is not correct when used with labels | Major | capacity scheduler | Sunil G | Sunil G | +| [HADOOP-13368](https://issues.apache.org/jira/browse/HADOOP-13368) | DFSOpsCountStatistics$OpType#fromSymbol and s3a.Statistic#fromSymbol should be O(1) operation | Major | fs | Mingliang Liu | Mingliang Liu | +| [HADOOP-13212](https://issues.apache.org/jira/browse/HADOOP-13212) | Provide an option to set the socket buffers in S3AFileSystem | Minor | fs/s3 | Rajesh Balamohan | Rajesh Balamohan | +| [HDFS-10653](https://issues.apache.org/jira/browse/HDFS-10653) | Optimize conversion from path string to components | Major | hdfs | Daryn Sharp | Daryn Sharp | +| [HADOOP-13207](https://issues.apache.org/jira/browse/HADOOP-13207) | Specify FileSystem listStatus, listFiles and RemoteIterator | Major | documentation, fs | Steve Loughran | Steve Loughran | +| [HADOOP-13188](https://issues.apache.org/jira/browse/HADOOP-13188) | S3A file-create should throw error rather than overwrite directories | Minor | fs/s3 | Raymie Stata | Steve Loughran | +| [HDFS-10642](https://issues.apache.org/jira/browse/HDFS-10642) | TestLazyPersistReplicaRecovery#testDnRestartWithSavedReplicas fails intermittently | Major | datanode, test | Mingliang Liu | Mingliang Liu | +| [HDFS-10668](https://issues.apache.org/jira/browse/HDFS-10668) | TestDataNodeMXBean#testDataNodeMXBeanBlockCount fails intermittently | Major | test | Mingliang Liu | Mingliang Liu | +| [YARN-5434](https://issues.apache.org/jira/browse/YARN-5434) | Add -client\|server argument for graceful decom | Blocker | graceful | Robert Kanter | Robert Kanter | +| [HADOOP-13429](https://issues.apache.org/jira/browse/HADOOP-13429) | Dispose of unnecessary SASL servers | Major | ipc | Daryn Sharp | Daryn Sharp | +| [HADOOP-13426](https://issues.apache.org/jira/browse/HADOOP-13426) | More efficiently build IPC responses | Major | . | Daryn Sharp | Daryn Sharp | +| [HDFS-10656](https://issues.apache.org/jira/browse/HDFS-10656) | Optimize conversion of byte arrays back to path string | Major | hdfs | Daryn Sharp | Daryn Sharp | +| [HDFS-10674](https://issues.apache.org/jira/browse/HDFS-10674) | Optimize creating a full path from an inode | Major | hdfs | Daryn Sharp | Daryn Sharp | +| [YARN-5342](https://issues.apache.org/jira/browse/YARN-5342) | Improve non-exclusive node partition resource allocation in Capacity Scheduler | Major | . | Wangda Tan | Sunil G | +| [HADOOP-13438](https://issues.apache.org/jira/browse/HADOOP-13438) | Optimize IPC server protobuf decoding | Major | . | Daryn Sharp | Daryn Sharp | +| [HADOOP-13418](https://issues.apache.org/jira/browse/HADOOP-13418) | Fix javadoc warnings by JDK8 in hadoop-nfs package | Major | . | Kai Sasaki | Kai Sasaki | +| [HDFS-10724](https://issues.apache.org/jira/browse/HDFS-10724) | Document the caller context config keys | Minor | ipc, namenode | Mingliang Liu | Mingliang Liu | +| [HDFS-10678](https://issues.apache.org/jira/browse/HDFS-10678) | Documenting NNThroughputBenchmark tool | Major | benchmarks, test | Mingliang Liu | Mingliang Liu | +| [HDFS-10641](https://issues.apache.org/jira/browse/HDFS-10641) | TestBlockManager#testBlockReportQueueing fails intermittently | Major | namenode, test | Mingliang Liu | Daryn Sharp | +| [HADOOP-13324](https://issues.apache.org/jira/browse/HADOOP-13324) | s3a tests don't authenticate with S3 frankfurt (or other V4 auth only endpoints) | Major | fs/s3 | Steve Loughran | Steve Loughran | +| [HADOOP-13208](https://issues.apache.org/jira/browse/HADOOP-13208) | S3A listFiles(recursive=true) to do a bulk listObjects instead of walking the pseudo-tree of directories | Minor | fs/s3 | Steve Loughran | Steve Loughran | +| [HADOOP-13405](https://issues.apache.org/jira/browse/HADOOP-13405) | doc for “fs.s3a.acl.default” indicates incorrect values | Minor | fs/s3 | Shen Yinjie | Shen Yinjie | +| [HDFS-10711](https://issues.apache.org/jira/browse/HDFS-10711) | Optimize FSPermissionChecker group membership check | Major | hdfs | Daryn Sharp | Daryn Sharp | +| [HADOOP-13252](https://issues.apache.org/jira/browse/HADOOP-13252) | Tune S3A provider plugin mechanism | Minor | fs/s3 | Steve Loughran | Steve Loughran | +| [HADOOP-13446](https://issues.apache.org/jira/browse/HADOOP-13446) | Support running isolated unit tests separate from AWS integration tests. | Major | fs/s3 | Chris Nauroth | Chris Nauroth | +| [HDFS-10762](https://issues.apache.org/jira/browse/HDFS-10762) | Pass IIP for file status related methods | Major | hdfs | Daryn Sharp | Daryn Sharp | +| [HDFS-10772](https://issues.apache.org/jira/browse/HDFS-10772) | Reduce byte/string conversions for get listing | Major | hdfs | Daryn Sharp | Daryn Sharp | +| [YARN-3940](https://issues.apache.org/jira/browse/YARN-3940) | Application moveToQueue should check NodeLabel permission | Major | resourcemanager | Bibin A Chundatt | Bibin A Chundatt | +| [HDFS-10768](https://issues.apache.org/jira/browse/HDFS-10768) | Optimize mkdir ops | Major | hdfs | Daryn Sharp | Daryn Sharp | +| [HDFS-10655](https://issues.apache.org/jira/browse/HDFS-10655) | Fix path related byte array conversion bugs | Major | hdfs | Daryn Sharp | Daryn Sharp | +| [HDFS-10662](https://issues.apache.org/jira/browse/HDFS-10662) | Optimize UTF8 string/byte conversions | Major | hdfs | Daryn Sharp | Daryn Sharp | +| [HADOOP-13547](https://issues.apache.org/jira/browse/HADOOP-13547) | Optimize IPC client protobuf decoding | Major | . | Daryn Sharp | Daryn Sharp | +| [HADOOP-13549](https://issues.apache.org/jira/browse/HADOOP-13549) | Eliminate intermediate buffer for server-side PB encoding | Major | ipc | Daryn Sharp | Daryn Sharp | +| [HADOOP-13447](https://issues.apache.org/jira/browse/HADOOP-13447) | Refactor S3AFileSystem to support introduction of separate metadata repository and tests. | Major | fs/s3 | Chris Nauroth | Chris Nauroth | +| [HADOOP-13541](https://issues.apache.org/jira/browse/HADOOP-13541) | explicitly declare the Joda time version S3A depends on | Minor | build, fs/s3 | Steve Loughran | Steve Loughran | +| [YARN-5566](https://issues.apache.org/jira/browse/YARN-5566) | Client-side NM graceful decom is not triggered when jobs finish | Major | nodemanager | Robert Kanter | Robert Kanter | +| [HADOOP-10940](https://issues.apache.org/jira/browse/HADOOP-10940) | RPC client does no bounds checking of responses | Critical | ipc | Daryn Sharp | Daryn Sharp | +| [HADOOP-13540](https://issues.apache.org/jira/browse/HADOOP-13540) | improve section on troubleshooting s3a auth problems | Minor | documentation, fs/s3 | Steve Loughran | Steve Loughran | +| [HDFS-8818](https://issues.apache.org/jira/browse/HDFS-8818) | Allow Balancer to run faster | Major | balancer & mover | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze | +| [HDFS-10673](https://issues.apache.org/jira/browse/HDFS-10673) | Optimize FSPermissionChecker's internal path usage | Major | hdfs | Daryn Sharp | Daryn Sharp | +| [HADOOP-13546](https://issues.apache.org/jira/browse/HADOOP-13546) | Override equals and hashCode to avoid connection leakage | Major | ipc | Xiaobing Zhou | Xiaobing Zhou | +| [HDFS-10744](https://issues.apache.org/jira/browse/HDFS-10744) | Internally optimize path component resolution | Major | hdfs | Daryn Sharp | Daryn Sharp | +| [HDFS-10805](https://issues.apache.org/jira/browse/HDFS-10805) | Reduce runtime for append test | Minor | test | Gergely Novák | Gergely Novák | +| [HADOOP-13544](https://issues.apache.org/jira/browse/HADOOP-13544) | JDiff reports unncessarily show unannotated APIs and cause confusion while our javadocs only show annotated and public APIs | Blocker | . | Vinod Kumar Vavilapalli | Vinod Kumar Vavilapalli | +| [HDFS-10779](https://issues.apache.org/jira/browse/HDFS-10779) | Rename does not need to re-solve destination | Major | hdfs | Daryn Sharp | Daryn Sharp | +| [HDFS-10892](https://issues.apache.org/jira/browse/HDFS-10892) | Add unit tests for HDFS command 'dfs -tail' and 'dfs -stat' | Major | fs, shell, test | Mingliang Liu | Mingliang Liu | +| [HADOOP-13599](https://issues.apache.org/jira/browse/HADOOP-13599) | s3a close() to be non-synchronized, so avoid risk of deadlock on shutdown | Major | fs/s3 | Steve Loughran | Steve Loughran | +| [HADOOP-12974](https://issues.apache.org/jira/browse/HADOOP-12974) | Create a CachingGetSpaceUsed implementation that uses df | Major | . | Elliott Clark | Elliott Clark | +| [HDFS-10851](https://issues.apache.org/jira/browse/HDFS-10851) | FSDirStatAndListingOp: stop passing path as string | Major | hdfs | Daryn Sharp | Daryn Sharp | +| [HDFS-10619](https://issues.apache.org/jira/browse/HDFS-10619) | Cache path in InodesInPath | Major | hdfs | Daryn Sharp | Daryn Sharp | +| [HDFS-10934](https://issues.apache.org/jira/browse/HDFS-10934) | TestDFSShell.testStat fails intermittently | Major | test | Eric Badger | Eric Badger | +| [HADOOP-13674](https://issues.apache.org/jira/browse/HADOOP-13674) | S3A can provide a more detailed error message when accessing a bucket through an incorrect S3 endpoint. | Minor | fs/s3 | Chris Nauroth | Chris Nauroth | +| [HDFS-10956](https://issues.apache.org/jira/browse/HDFS-10956) | Remove rename/delete performance penalty when not using snapshots | Major | hdfs | Daryn Sharp | Daryn Sharp | +| [HDFS-10896](https://issues.apache.org/jira/browse/HDFS-10896) | Move lock logging logic from FSNamesystem into FSNamesystemLock | Major | namenode | Erik Krogen | Erik Krogen | +| [HDFS-10745](https://issues.apache.org/jira/browse/HDFS-10745) | Directly resolve paths into INodesInPath | Major | hdfs | Daryn Sharp | Daryn Sharp | +| [HDFS-10955](https://issues.apache.org/jira/browse/HDFS-10955) | Pass IIP for FSDirAttr methods | Major | hdfs | Daryn Sharp | Daryn Sharp | +| [HADOOP-12977](https://issues.apache.org/jira/browse/HADOOP-12977) | s3a to handle delete("/", true) robustly | Minor | fs/s3 | Steve Loughran | Steve Loughran | +| [HDFS-10939](https://issues.apache.org/jira/browse/HDFS-10939) | Reduce performance penalty of encryption zones | Major | hdfs | Daryn Sharp | Daryn Sharp | +| [HADOOP-13692](https://issues.apache.org/jira/browse/HADOOP-13692) | hadoop-aws should declare explicit dependency on Jackson 2 jars to prevent classpath conflicts. | Minor | fs/s3 | Chris Nauroth | Chris Nauroth | +| [HDFS-10979](https://issues.apache.org/jira/browse/HDFS-10979) | Pass IIP for FSDirDeleteOp methods | Major | hdfs | Daryn Sharp | Daryn Sharp | +| [HDFS-10980](https://issues.apache.org/jira/browse/HDFS-10980) | Optimize check for existence of parent directory | Major | hdfs | Daryn Sharp | Daryn Sharp | +| [HDFS-10988](https://issues.apache.org/jira/browse/HDFS-10988) | Refactor TestBalancerBandwidth | Major | balancer & mover, test | Brahma Reddy Battula | Brahma Reddy Battula | +| [HDFS-10985](https://issues.apache.org/jira/browse/HDFS-10985) | o.a.h.ha.TestZKFailoverController should not use fixed time sleep before assertions | Minor | ha, test | Mingliang Liu | Mingliang Liu | +| [HDFS-10972](https://issues.apache.org/jira/browse/HDFS-10972) | Add unit test for HDFS command 'dfsadmin -getDatanodeInfo' | Major | fs, shell, test | Xiaobing Zhou | Xiaobing Zhou | +| [HDFS-10965](https://issues.apache.org/jira/browse/HDFS-10965) | Add unit test for HDFS command 'dfsadmin -printTopology' | Major | fs, shell, test | Xiaobing Zhou | Xiaobing Zhou | +| [HDFS-11008](https://issues.apache.org/jira/browse/HDFS-11008) | Change unit test for testing parsing "-source" parameter in Balancer CLI | Major | test | Mingliang Liu | Mingliang Liu | +| [HADOOP-13419](https://issues.apache.org/jira/browse/HADOOP-13419) | Fix javadoc warnings by JDK8 in hadoop-common package | Major | . | Kai Sasaki | Kai Sasaki | +| [HDFS-10922](https://issues.apache.org/jira/browse/HDFS-10922) | Adding additional unit tests for Trash (II) | Major | test | Xiaoyu Yao | Weiwei Yang | +| [HDFS-10906](https://issues.apache.org/jira/browse/HDFS-10906) | Add unit tests for Trash with HDFS encryption zones | Major | encryption | Xiaoyu Yao | Hanisha Koneru | +| [HADOOP-13735](https://issues.apache.org/jira/browse/HADOOP-13735) | ITestS3AFileContextStatistics.testStatistics() failing | Minor | fs/s3 | Steve Loughran | Pieter Reuse | +| [HDFS-10998](https://issues.apache.org/jira/browse/HDFS-10998) | Add unit tests for HDFS command 'dfsadmin -fetchImage' in HA | Major | test | Xiaobing Zhou | Xiaobing Zhou | +| [HADOOP-13727](https://issues.apache.org/jira/browse/HADOOP-13727) | S3A: Reduce high number of connections to EC2 Instance Metadata Service caused by InstanceProfileCredentialsProvider. | Minor | fs/s3 | Rajesh Balamohan | Chris Nauroth | +| [HADOOP-12774](https://issues.apache.org/jira/browse/HADOOP-12774) | s3a should use UGI.getCurrentUser.getShortname() for username | Major | fs/s3 | Steve Loughran | Steve Loughran | +| [HADOOP-13309](https://issues.apache.org/jira/browse/HADOOP-13309) | Document S3A known limitations in file ownership and permission model. | Minor | fs/s3 | Chris Nauroth | Chris Nauroth | +| [HDFS-11011](https://issues.apache.org/jira/browse/HDFS-11011) | Add unit tests for HDFS command 'dfsadmin -set/clrSpaceQuota' | Major | hdfs-client | Xiaobing Zhou | Xiaobing Zhou | +| [HADOOP-13614](https://issues.apache.org/jira/browse/HADOOP-13614) | Purge some superfluous/obsolete S3 FS tests that are slowing test runs down | Minor | fs/s3, test | Steve Loughran | Steve Loughran | +| [HADOOP-10597](https://issues.apache.org/jira/browse/HADOOP-10597) | RPC Server signals backoff to clients when all request queues are full | Major | . | Ming Ma | Ming Ma | +| [HADOOP-13680](https://issues.apache.org/jira/browse/HADOOP-13680) | fs.s3a.readahead.range to use getLongBytes | Major | fs/s3 | Steve Loughran | Abhishek Modi | +| [HDFS-11030](https://issues.apache.org/jira/browse/HDFS-11030) | TestDataNodeVolumeFailure#testVolumeFailure is flaky (though passing) | Major | datanode, test | Mingliang Liu | Mingliang Liu | +| [HDFS-10997](https://issues.apache.org/jira/browse/HDFS-10997) | Reduce number of path resolving methods | Major | namenode | Daryn Sharp | Daryn Sharp | +| [HDFS-11065](https://issues.apache.org/jira/browse/HDFS-11065) | Add space quota tests for heterogenous storages | Major | hdfs | Xiaobing Zhou | Xiaobing Zhou | +| [HDFS-11031](https://issues.apache.org/jira/browse/HDFS-11031) | Add additional unit test for DataNode startup behavior when volumes fail | Major | datanode, test | Mingliang Liu | Mingliang Liu | +| [HADOOP-10300](https://issues.apache.org/jira/browse/HADOOP-10300) | Allowed deferred sending of call responses | Major | ipc | Daryn Sharp | Daryn Sharp | +| [HDFS-11076](https://issues.apache.org/jira/browse/HDFS-11076) | Add unit test for extended Acls | Major | test | Chen Liang | Chen Liang | +| [HDFS-11085](https://issues.apache.org/jira/browse/HDFS-11085) | Add unit test for NameNode failing to start when name dir is unwritable | Major | namenode, test | Mingliang Liu | Xiaobing Zhou | +| [YARN-5802](https://issues.apache.org/jira/browse/YARN-5802) | updateApplicationPriority api in scheduler should ensure to re-insert app to correct ordering policy | Critical | capacity scheduler | Bibin A Chundatt | Bibin A Chundatt | +| [HDFS-11083](https://issues.apache.org/jira/browse/HDFS-11083) | Add unit test for DFSAdmin -report command | Major | shell, test | Mingliang Liu | Xiaobing Zhou | +| [YARN-4498](https://issues.apache.org/jira/browse/YARN-4498) | Application level node labels stats to be available in REST | Major | api, client, resourcemanager | Bibin A Chundatt | Bibin A Chundatt | +| [HDFS-11122](https://issues.apache.org/jira/browse/HDFS-11122) | TestDFSAdmin#testReportCommand fails due to timed out | Minor | test | Yiqun Lin | Yiqun Lin | +| [HDFS-10872](https://issues.apache.org/jira/browse/HDFS-10872) | Add MutableRate metrics for FSNamesystemLock operations | Major | namenode | Erik Krogen | Erik Krogen | +| [HDFS-11105](https://issues.apache.org/jira/browse/HDFS-11105) | TestRBWBlockInvalidation#testRWRInvalidation fails intermittently | Major | namenode, test | Yiqun Lin | Yiqun Lin | +| [HADOOP-13822](https://issues.apache.org/jira/browse/HADOOP-13822) | Use GlobalStorageStatistics.INSTANCE.reset() at FileSystem#clearStatistics() | Major | fs | Brahma Reddy Battula | Brahma Reddy Battula | +| [HADOOP-11601](https://issues.apache.org/jira/browse/HADOOP-11601) | Enhance FS spec & tests to mandate FileStatus.getBlocksize() \>0 for non-empty files | Minor | fs, test | Steve Loughran | Steve Loughran | +| [HADOOP-13801](https://issues.apache.org/jira/browse/HADOOP-13801) | regression: ITestS3AMiniYarnCluster failing | Major | fs/s3 | Steve Loughran | Steve Loughran | +| [HADOOP-12804](https://issues.apache.org/jira/browse/HADOOP-12804) | Read Proxy Password from Credential Providers in S3 FileSystem | Minor | fs/s3 | Larry McCay | Larry McCay | +| [HADOOP-13823](https://issues.apache.org/jira/browse/HADOOP-13823) | s3a rename: fail if dest file exists | Blocker | fs/s3 | Steve Loughran | Steve Loughran | +| [HADOOP-13857](https://issues.apache.org/jira/browse/HADOOP-13857) | S3AUtils.translateException to map (wrapped) InterruptedExceptions to InterruptedIOEs | Major | fs/s3 | Steve Loughran | Steve Loughran | +| [HADOOP-13855](https://issues.apache.org/jira/browse/HADOOP-13855) | Fix a couple of the s3a statistic names to be consistent with the rest | Major | fs/s3 | Steve Loughran | Steve Loughran | +| [HADOOP-13257](https://issues.apache.org/jira/browse/HADOOP-13257) | Improve Azure Data Lake contract tests. | Major | fs/adl | Chris Nauroth | Vishwajeet Dusane | +| [YARN-4390](https://issues.apache.org/jira/browse/YARN-4390) | Do surgical preemption based on reserved container in CapacityScheduler | Major | capacity scheduler | Eric Payne | Wangda Tan | +| [HDFS-8630](https://issues.apache.org/jira/browse/HDFS-8630) | WebHDFS : Support get/set/unset StoragePolicy | Major | webhdfs | nijel | Surendra Singh Lilhore | +| [HADOOP-13871](https://issues.apache.org/jira/browse/HADOOP-13871) | ITestS3AInputStreamPerformance.testTimeToOpenAndReadWholeFileBlocks performance awful | Major | fs/s3 | Steve Loughran | Steve Loughran | +| [YARN-2009](https://issues.apache.org/jira/browse/YARN-2009) | CapacityScheduler: Add intra-queue preemption for app priority support | Major | capacityscheduler | Devaraj K | Sunil G | +| [YARN-4844](https://issues.apache.org/jira/browse/YARN-4844) | Add getMemorySize/getVirtualCoresSize to o.a.h.y.api.records.Resource | Blocker | api | Wangda Tan | Wangda Tan | +| [YARN-4990](https://issues.apache.org/jira/browse/YARN-4990) | Re-direction of a particular log file within in a container in NM UI does not redirect properly to Log Server ( history ) on container completion | Major | . | Hitesh Shah | Xuan Gong | +| [YARN-3866](https://issues.apache.org/jira/browse/YARN-3866) | AM-RM protocol changes to support container resizing | Blocker | api | MENG DING | MENG DING | +| [HADOOP-13336](https://issues.apache.org/jira/browse/HADOOP-13336) | S3A to support per-bucket configuration | Major | fs/s3 | Steve Loughran | Steve Loughran | +| [HDFS-8377](https://issues.apache.org/jira/browse/HDFS-8377) | Support HTTP/2 in datanode | Major | . | Duo Zhang | Duo Zhang | +| [HADOOP-14019](https://issues.apache.org/jira/browse/HADOOP-14019) | fix some typos in the s3a docs | Minor | documentation, fs/s3 | Steve Loughran | Steve Loughran | +| [HADOOP-14081](https://issues.apache.org/jira/browse/HADOOP-14081) | S3A: Consider avoiding array copy in S3ABlockOutputStream (ByteArrayBlock) | Minor | fs/s3 | Rajesh Balamohan | Rajesh Balamohan | +| [YARN-6143](https://issues.apache.org/jira/browse/YARN-6143) | Fix incompatible issue caused by YARN-3583 | Blocker | rolling upgrade | Wangda Tan | Sunil G | +| [HADOOP-14113](https://issues.apache.org/jira/browse/HADOOP-14113) | review ADL Docs | Minor | documentation, fs/adl | Steve Loughran | Steve Loughran | +| [HADOOP-14138](https://issues.apache.org/jira/browse/HADOOP-14138) | Remove S3A ref from META-INF service discovery, rely on existing core-default entry | Critical | fs/s3 | Steve Loughran | Steve Loughran | +| [HADOOP-14123](https://issues.apache.org/jira/browse/HADOOP-14123) | Remove misplaced ADL service provider config file for FileSystem | Minor | fs/adl | John Zhuge | John Zhuge | +| [HADOOP-14153](https://issues.apache.org/jira/browse/HADOOP-14153) | ADL module has messed doc structure | Major | fs/adl | Mingliang Liu | Mingliang Liu | +| [HADOOP-14173](https://issues.apache.org/jira/browse/HADOOP-14173) | Remove unused AdlConfKeys#ADL\_EVENTS\_TRACKING\_SOURCE | Trivial | fs/adl | John Zhuge | John Zhuge | + + +### OTHER: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [YARN-3357](https://issues.apache.org/jira/browse/YARN-3357) | Move TestFifoScheduler to FIFO package | Major | scheduler, test | Rohith Sharma K S | Rohith Sharma K S | +| [HADOOP-11814](https://issues.apache.org/jira/browse/HADOOP-11814) | Reformat hadoop-annotations, o.a.h.classification.tools | Minor | . | Li Lu | Li Lu | +| [MAPREDUCE-6388](https://issues.apache.org/jira/browse/MAPREDUCE-6388) | Remove deprecation warnings from JobHistoryServer classes | Minor | jobhistoryserver | Ray Chiang | Ray Chiang | +| [YARN-3026](https://issues.apache.org/jira/browse/YARN-3026) | Move application-specific container allocation logic from LeafQueue to FiCaSchedulerApp | Major | capacityscheduler | Wangda Tan | Wangda Tan | +| [HDFS-8938](https://issues.apache.org/jira/browse/HDFS-8938) | Extract BlockToMarkCorrupt and ReplicationWork as standalone classes from BlockManager | Major | . | Mingliang Liu | Mingliang Liu | +| [HDFS-9027](https://issues.apache.org/jira/browse/HDFS-9027) | Refactor o.a.h.hdfs.DataStreamer#isLazyPersist() method | Major | . | Mingliang Liu | Mingliang Liu | +| [MAPREDUCE-6477](https://issues.apache.org/jira/browse/MAPREDUCE-6477) | Replace usage of deprecated NameNode.DEFAULT\_PORT in TestFileSystem | Major | . | Mingliang Liu | Mingliang Liu | +| [MAPREDUCE-6483](https://issues.apache.org/jira/browse/MAPREDUCE-6483) | Replace deprecated method NameNode.getUri() with DFSUtilClient.getNNUri() in TestMRCredentials | Major | test | Mingliang Liu | Mingliang Liu | +| [HDFS-9130](https://issues.apache.org/jira/browse/HDFS-9130) | Use GenericTestUtils#setLogLevel to the logging level | Major | . | Mingliang Liu | Mingliang Liu | +| [HADOOP-12446](https://issues.apache.org/jira/browse/HADOOP-12446) | Undeprecate createNonRecursive() | Major | . | Ted Yu | Ted Yu | +| [HDFS-8979](https://issues.apache.org/jira/browse/HDFS-8979) | Clean up checkstyle warnings in hadoop-hdfs-client module | Major | . | Mingliang Liu | Mingliang Liu | +| [HADOOP-11791](https://issues.apache.org/jira/browse/HADOOP-11791) | Update src/site/markdown/releases to include old versions of Hadoop | Major | build, documentation | Allen Wittenauer | Allen Wittenauer | +| [HADOOP-12514](https://issues.apache.org/jira/browse/HADOOP-12514) | Make static fields in GenericTestUtils for assertExceptionContains() package-private and final | Minor | test | Mingliang Liu | Mingliang Liu | +| [HDFS-9377](https://issues.apache.org/jira/browse/HDFS-9377) | Fix findbugs warnings in FSDirSnapshotOp | Major | namenode | Mingliang Liu | Mingliang Liu | +| [HADOOP-12567](https://issues.apache.org/jira/browse/HADOOP-12567) | NPE in SaslRpcServer | Major | . | Sergey Shelukhin | Sergey Shelukhin | +| [YARN-4653](https://issues.apache.org/jira/browse/YARN-4653) | Document YARN security model from the perspective of Application Developers | Major | site | Steve Loughran | Steve Loughran | +| [HDFS-10200](https://issues.apache.org/jira/browse/HDFS-10200) | Docs for WebHDFS still describe GETDELEGATIONTOKENS operation | Trivial | documentation | Wellington Chevreuil | Wellington Chevreuil | +| [HDFS-9353](https://issues.apache.org/jira/browse/HDFS-9353) | Code and comment mismatch in JavaKeyStoreProvider | Trivial | . | nijel | Andras Bokor | +| [HDFS-10984](https://issues.apache.org/jira/browse/HDFS-10984) | Expose nntop output as metrics | Major | namenode | Siddharth Wagle | Siddharth Wagle | +| [YARN-5704](https://issues.apache.org/jira/browse/YARN-5704) | Provide config knobs to control enabling/disabling new/work in progress features in container-executor | Major | yarn | Sidharta Seethana | Sidharta Seethana | +| [HADOOP-14091](https://issues.apache.org/jira/browse/HADOOP-14091) | AbstractFileSystem implementaion for 'wasbs' scheme | Major | fs/azure | Varada Hemeswari | Varada Hemeswari | +| [YARN-6274](https://issues.apache.org/jira/browse/YARN-6274) | Documentation refers to incorrect nodemanager health checker interval property | Trivial | documentation | Charles Zhang | Weiwei Yang | + + diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/2.8.0/RELEASENOTES.2.8.0.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/2.8.0/RELEASENOTES.2.8.0.md new file mode 100644 index 00000000000..e3c19b6c644 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/2.8.0/RELEASENOTES.2.8.0.md @@ -0,0 +1,1105 @@ + + +# "Apache Hadoop" 2.8.0 Release Notes + +These release notes cover new developer and user-facing incompatibilities, important issues, features, and major improvements. + + +--- + +* [HADOOP-7713](https://issues.apache.org/jira/browse/HADOOP-7713) | *Trivial* | **dfs -count -q should label output column** + +Added -v option to fs -count command to display a header record in the report. + + +--- + +* [HADOOP-8934](https://issues.apache.org/jira/browse/HADOOP-8934) | *Minor* | **Shell command ls should include sort options** + +Options to sort output of fs -ls comment: -t (mtime), -S (size), -u (atime), -r (reverse) + + +--- + +* [HADOOP-11226](https://issues.apache.org/jira/browse/HADOOP-11226) | *Major* | **Add a configuration to set ipc.Client's traffic class with IPTOS\_LOWDELAY\|IPTOS\_RELIABILITY** + +Use low latency TCP connections for hadoop IPC + + +--- + +* [HADOOP-9477](https://issues.apache.org/jira/browse/HADOOP-9477) | *Major* | **Add posixGroups support for LDAP groups mapping service** + +Add posixGroups support for LDAP groups mapping service. The change in LDAPGroupMapping is compatible with previous scenario. In LDAP, the group mapping between {{posixAccount}} and {{posixGroup}} is different from the general LDAPGroupMapping, one of the differences is the {{"memberUid"}} will be used to mapping {{posixAccount}} and {{posixGroup}}. The feature will handle the mapping in internal when configuration {{hadoop.security.group.mapping.ldap.search.filter.user}} is set as "posixAccount" and {{hadoop.security.group.mapping.ldap.search.filter.group}} is "posixGroup". + + +--- + +* [YARN-3241](https://issues.apache.org/jira/browse/YARN-3241) | *Major* | **FairScheduler handles "invalid" queue names inconsistently** + +FairScheduler does not allow queue names with leading or tailing spaces or empty sub-queue names anymore. + + +--- + +* [HDFS-7501](https://issues.apache.org/jira/browse/HDFS-7501) | *Major* | **TransactionsSinceLastCheckpoint can be negative on SBNs** + +Fixed a bug where the StandbyNameNode's TransactionsSinceLastCheckpoint metric may slide into a negative number after every subsequent checkpoint. + + +--- + +* [HADOOP-11660](https://issues.apache.org/jira/browse/HADOOP-11660) | *Minor* | **Add support for hardware crc of HDFS checksums on ARM aarch64 architecture** + +Add support for aarch64 CRC instructions + + +--- + +* [HADOOP-11731](https://issues.apache.org/jira/browse/HADOOP-11731) | *Major* | **Rework the changelog and releasenotes** + + +* The release notes now only contains JIRA issues with incompatible changes and actual release notes. The generated format has been changed from HTML to markdown. + +* The changelog is now automatically generated from data stored in JIRA rather than manually maintained. The format has been changed from pure text to markdown as well as containing more of the information that was previously stored in the release notes. + +* In order to generate the changes file, python must be installed. + +* New -Preleasedocs profile added to maven in order to trigger this functionality. + + +--- + +* [YARN-3365](https://issues.apache.org/jira/browse/YARN-3365) | *Major* | **Add support for using the 'tc' tool via container-executor** + +Adding support for using the 'tc' tool in batch mode via container-executor. This is a prerequisite for traffic-shaping functionality that is necessary to support outbound bandwidth as a resource in YARN. + + +--- + +* [YARN-3443](https://issues.apache.org/jira/browse/YARN-3443) | *Major* | **Create a 'ResourceHandler' subsystem to ease addition of support for new resource types on the NM** + +The current cgroups implementation is closely tied to supporting CPU as a resource . This patch separates out CGroups implementation into a reusable class as well as provides a simple ResourceHandler subsystem that will enable us to add support for new resource types on the NM - e.g Network, Disk etc. + + +--- + +* [HDFS-6666](https://issues.apache.org/jira/browse/HDFS-6666) | *Minor* | **Abort NameNode and DataNode startup if security is enabled but block access token is not enabled.** + +NameNode and DataNode now abort during startup if attempting to run in secure mode, but block access tokens are not enabled by setting configuration property dfs.block.access.token.enable to true in hdfs-site.xml. Previously, this case logged a warning, because this would be an insecure configuration. + + +--- + +* [YARN-3021](https://issues.apache.org/jira/browse/YARN-3021) | *Major* | **YARN's delegation-token handling disallows certain trust setups to operate properly over DistCp** + +ResourceManager renews delegation tokens for applications. This behavior has been changed to renew tokens only if the token's renewer is a non-empty string. MapReduce jobs can instruct ResourceManager to skip renewal of tokens obtained from certain hosts by specifying the hosts with configuration mapreduce.job.hdfs-servers.token-renewal.exclude=\,\,..,\. + + +--- + +* [HADOOP-11746](https://issues.apache.org/jira/browse/HADOOP-11746) | *Major* | **rewrite test-patch.sh** + + +* test-patch.sh now has new output that is different than the previous versions +* test-patch.sh is now pluggable via the test-patch.d directory, with checkstyle and shellcheck tests included +* JIRA comments now use much more markup to improve readability +* test-patch.sh now supports either a file name, a URL, or a JIRA issue as input in developer mode +* If part of the patch testing code is changed, test-patch.sh will now attempt to re-executing itself using the new version. +* Some logic to try and reduce the amount of unnecessary tests. For example, patches that only modify markdown should not run the Java compilation tests. +* Plugins for checkstyle, shellcheck, and whitespace now execute as necessary. +* New test code for mvn site +* A breakdown of the times needed to execute certain blocks as well as a total runtime is now reported to assist in fixing long running tests and optimize the entire process. +* Several new options + * --resetrepo will put test-patch.sh in destructive mode, similar to a normal Jenkins run + * --testlist allows one to provide a comma delimited list of test subsystems to forcibly execute + * --modulelist to provide a comma delimited list of module tests to execute in addition to the ones that are automatically detected + * --offline mode to attempt to stop connecting to the Internet for certain operations +* test-patch.sh now defaults to the POSIX equivalents on Solaris and Illumos-based operating systems +* shelldocs.py may be used to generate test-patch.sh API information +* FindBugs output is now listed on the JIRA comment +* lots of general code cleanup, including attempts to remove any local state files to reduce potential race conditions +* Some logic to determine if a patch is for a given major branch using several strategies as well as a particular git ref (using git+ref as part of the name). +* Some logic to determine if a patch references a particular JIRA issue. +* Unit tests are only flagged as necessary with native or Java code, since Hadoop has no framework in place yet for other types of unit tests. +* test-patch now exits with a failure status if problems arise trying to do git checkouts. Previously the exit code was success. + + +--- + +* [YARN-3366](https://issues.apache.org/jira/browse/YARN-3366) | *Major* | **Outbound network bandwidth : classify/shape traffic originating from YARN containers** + +1) A TrafficController class that provides an implementation for traffic shaping using tc. +2) A ResourceHandler implementation for OutboundBandwidth as a resource - isolation/enforcement using cgroups and tc. + + +--- + +* [HADOOP-11861](https://issues.apache.org/jira/browse/HADOOP-11861) | *Major* | **test-patch.sh rewrite addendum patch** + + +* --build-native=false should work now +* --branch option lets one specify a branch to test against on the command line +* On certain Jenkins machines, the artifact directory sometimes gets deleted from outside the test-patch script. There is now some code to try to detect, alert, and quick exit if that happens. +* Various semi-critical output and bug fixes + + +--- + +* [HADOOP-11843](https://issues.apache.org/jira/browse/HADOOP-11843) | *Major* | **Make setting up the build environment easier** + +Includes a docker based solution for setting up a build environment with minimal effort. + + +--- + +* [HADOOP-11813](https://issues.apache.org/jira/browse/HADOOP-11813) | *Minor* | **releasedocmaker.py should use today's date instead of unreleased** + +Use today instead of 'Unreleased' in releasedocmaker.py when --usetoday is given as an option. + + +--- + +* [HDFS-8226](https://issues.apache.org/jira/browse/HDFS-8226) | *Blocker* | **Non-HA rollback compatibility broken** + +Non-HA rollback steps have been changed. Run the rollback command on the namenode (`bin/hdfs namenode -rollback`) before starting cluster with '-rollback' option using (sbin/start-dfs.sh -rollback). + + +--- + +* [HDFS-6888](https://issues.apache.org/jira/browse/HDFS-6888) | *Major* | **Allow selectively audit logging ops** + +Specific HDFS ops can be selectively excluded from audit logging via 'dfs.namenode.audit.log.debug.cmdlist' configuration. + + +--- + +* [HDFS-8157](https://issues.apache.org/jira/browse/HDFS-8157) | *Major* | **Writes to RAM DISK reserve locked memory for block files** + +This change requires setting the dfs.datanode.max.locked.memory configuration key to use the HDFS Lazy Persist feature. Its value limits the combined off-heap memory for blocks in RAM via caching and lazy persist writes. + + +--- + +* [HADOOP-11772](https://issues.apache.org/jira/browse/HADOOP-11772) | *Major* | **RPC Invoker relies on static ClientCache which has synchronized(this) blocks** + +The Client#call() methods that are deprecated since 0.23 have been removed. + + +--- + +* [YARN-3684](https://issues.apache.org/jira/browse/YARN-3684) | *Major* | **Change ContainerExecutor's primary lifecycle methods to use a more extensible mechanism for passing information.** + +Modifying key methods in ContainerExecutor to use context objects instead of an argument list. This is more extensible and less brittle. + + +--- + +* [YARN-2336](https://issues.apache.org/jira/browse/YARN-2336) | *Major* | **Fair scheduler REST api returns a missing '[' bracket JSON for deep queue tree** + +Fix FairScheduler's REST api returns a missing '[' blacket JSON for childQueues. + + +--- + +* [HDFS-8486](https://issues.apache.org/jira/browse/HDFS-8486) | *Blocker* | **DN startup may cause severe data loss** + + +Public service notice: +* Every restart of a 2.6.x or 2.7.0 DN incurs a risk of unwanted block deletion. +* Apply this patch if you are running a pre-2.7.1 release. + + +--- + +* [HDFS-8270](https://issues.apache.org/jira/browse/HDFS-8270) | *Major* | **create() always retried with hardcoded timeout when file already exists with open lease** + +Proxy level retries will not be done on AlreadyBeingCreatedExeption for create() op. + + +--- + +* [YARN-41](https://issues.apache.org/jira/browse/YARN-41) | *Major* | **The RM should handle the graceful shutdown of the NM.** + +The behavior of shutdown a NM could be different (if NM work preserving is not enabled): NM will unregister to RM immediately rather than waiting for timeout to be LOST. A new status of NodeStatus - SHUTDOWN is involved which could affect UI, CLI and ClusterMetrics for node's status. + + +--- + +* [HADOOP-7139](https://issues.apache.org/jira/browse/HADOOP-7139) | *Major* | **Allow appending to existing SequenceFiles** + +Existing sequence files can be appended. + + +--- + +* [HDFS-8582](https://issues.apache.org/jira/browse/HDFS-8582) | *Minor* | **Support getting a list of reconfigurable config properties and do not generate spurious reconfig warnings** + +Add a new option "properties" to the "dfsadmin -reconfig" command to get a list of reconfigurable properties. + + +--- + +* [HDFS-6564](https://issues.apache.org/jira/browse/HDFS-6564) | *Major* | **Use slf4j instead of common-logging in hdfs-client** + +Users may need special attention for this change while upgrading to this version. Previously hdfs client was using commons-logging as the logging framework. With this change it will use slf4j framework. For more details about slf4j, please see: http://www.slf4j.org/manual.html. Also, org.apache.hadoop.hdfs.protocol.CachePoolInfo#LOG public static member variable has been removed as it is not used anywhere. Users need to correct their code if any one has a reference to this variable. One can retrieve the named logger via the logging framework of their choice directly like, org.slf4j.Logger LOG = org.slf4j.LoggerFactory.getLogger(org.apache.hadoop.hdfs.protocol.CachePoolInfo.class); + + +--- + +* [YARN-3823](https://issues.apache.org/jira/browse/YARN-3823) | *Minor* | **Fix mismatch in default values for yarn.scheduler.maximum-allocation-vcores property** + +Default value for 'yarn.scheduler.maximum-allocation-vcores' changed from 32 to 4. + + +--- + +* [HADOOP-5732](https://issues.apache.org/jira/browse/HADOOP-5732) | *Minor* | **Add SFTP FileSystem** + +Added SFTP filesystem by using the JSch library. + + +--- + +* [YARN-3069](https://issues.apache.org/jira/browse/YARN-3069) | *Major* | **Document missing properties in yarn-default.xml** + +Documented missing properties and added the regression test to verify that there are no missing properties in yarn-default.xml. + + +--- + +* [MAPREDUCE-6427](https://issues.apache.org/jira/browse/MAPREDUCE-6427) | *Minor* | **Fix typo in JobHistoryEventHandler** + +There is a typo in the event string "WORKFLOW\_ID" (as "WORKLFOW\_ID"). The branch-2 change will publish both event strings for compatibility with consumers, but the misspelled metric will be removed in trunk. + + +--- + +* [HDFS-7582](https://issues.apache.org/jira/browse/HDFS-7582) | *Major* | **Enforce maximum number of ACL entries separately per access and default.** + +Limit on Maximum number of ACL entries(32) will be enforced separately on access and default ACLs. So in total, max. 64 ACL entries can be present in a ACL spec. + + +--- + +* [HADOOP-12269](https://issues.apache.org/jira/browse/HADOOP-12269) | *Major* | **Update aws-sdk dependency to 1.10.6; move to aws-sdk-s3** + +The Maven dependency on aws-sdk has been changed to aws-sdk-s3 and the version bumped. Applications depending on transitive dependencies pulled in by aws-sdk and not aws-sdk-s3 might not work. + + +--- + +* [HADOOP-12352](https://issues.apache.org/jira/browse/HADOOP-12352) | *Trivial* | **Delay in checkpointing Trash can leave trash for 2 intervals before deleting** + +Fixes an Trash related issue wherein a delay in the periodic checkpointing of one user's directory causes the subsequent user directory checkpoints to carry a newer timestamp, thereby delaying their eventual deletion. + + +--- + +* [HDFS-8900](https://issues.apache.org/jira/browse/HDFS-8900) | *Major* | **Compact XAttrs to optimize memory footprint.** + +The config key "dfs.namenode.fs-limits.max-xattr-size" can no longer be set to a value of 0 (previously used to indicate unlimited) or a value greater than 32KB. This is a constraint on xattr size similar to many local filesystems. + + +--- + +* [HDFS-8890](https://issues.apache.org/jira/browse/HDFS-8890) | *Major* | **Allow admin to specify which blockpools the balancer should run on** + +Adds a new blockpools flag to the balancer. This allows admins to specify which blockpools the balancer will run on. +Usage: +-blockpools \ +The balancer will only run on blockpools included in this list. + + +--- + +* [YARN-4087](https://issues.apache.org/jira/browse/YARN-4087) | *Major* | **Followup fixes after YARN-2019 regarding RM behavior when state-store error occurs** + +Set YARN\_FAIL\_FAST to be false by default. If HA is enabled and if there's any state-store error, after the retry operation failed, we always transition RM to standby state. + + +--- + +* [HADOOP-12384](https://issues.apache.org/jira/browse/HADOOP-12384) | *Major* | **Add "-direct" flag option for fs copy so that user can choose not to create ".\_COPYING\_" file** + +An option '-d' added for all command-line copy commands to skip intermediate '.COPYING' file creation. + + +--- + +* [HDFS-8929](https://issues.apache.org/jira/browse/HDFS-8929) | *Major* | **Add a metric to expose the timestamp of the last journal** + +Exposed a metric 'LastJournalTimestamp' for JournalNode + + +--- + +* [HDFS-7116](https://issues.apache.org/jira/browse/HDFS-7116) | *Major* | **Add a command to get the balancer bandwidth** + +Exposed command "-getBalancerBandwidth" in dfsadmin to get the bandwidth of balancer. + + +--- + +* [HDFS-8829](https://issues.apache.org/jira/browse/HDFS-8829) | *Major* | **Make SO\_RCVBUF and SO\_SNDBUF size configurable for DataTransferProtocol sockets and allow configuring auto-tuning** + +HDFS-8829 introduces two new configuration settings: dfs.datanode.transfer.socket.send.buffer.size and dfs.datanode.transfer.socket.recv.buffer.size. These settings can be used to control the socket send buffer and receive buffer sizes respectively on the DataNode for client-DataNode and DataNode-DataNode connections. The default values of both settings are 128KB for backwards compatibility. For optimum performance it is recommended to set these values to zero to enable the OS networking stack to auto-tune buffer sizes. + + +--- + +* [YARN-313](https://issues.apache.org/jira/browse/YARN-313) | *Critical* | **Add Admin API for supporting node resource configuration in command line** + +After this patch, the feature to support NM resource dynamically configuration is completed, so that user can configure NM with new resource without bring NM down or decommissioned. +Two CLIs are provided to support update resources on individual node or a batch of nodes: +1. Update resource on single node: yarn rmadmin -updateNodeResource [NodeID] [MemSize] [vCores] +2. Update resource on a batch of nodes: yarn rmadmin -refreshNodesResources, that reflect nodes' resource configuration defined in dynamic-resources.xml which is loaded by RM dynamically (like capacity-scheduler.xml or fair-scheduler.xml). +The first version of configuration format is: +\ + \ + \yarn.resource.dynamic.nodes\ + \h1:1234\ + \ + \ + \yarn.resource.dynamic.h1:1234.vcores\ + \16\ + \ + \ + \yarn.resource.dynamic.h1:1234.memory\ + \1024\ + \ +\ + + +--- + +* [HADOOP-12416](https://issues.apache.org/jira/browse/HADOOP-12416) | *Major* | **Trash messages should be handled by Logger instead of being delivered on System.out** + +Now trash message is not printed to System.out. It is handled by Logger instead. + + +--- + +* [HDFS-9063](https://issues.apache.org/jira/browse/HDFS-9063) | *Major* | **Correctly handle snapshot path for getContentSummary** + +The jira made the following changes: +1. Fix a bug to exclude newly-created files from quota usage calculation for a snapshot path. +2. Number of snapshots is no longer counted as directory number in getContentSummary result. + + +--- + +* [HADOOP-12360](https://issues.apache.org/jira/browse/HADOOP-12360) | *Minor* | **Create StatsD metrics2 sink** + +Added StatsD metrics2 sink + + +--- + +* [HDFS-9013](https://issues.apache.org/jira/browse/HDFS-9013) | *Major* | **Deprecate NameNodeMXBean#getNNStarted in branch2 and remove from trunk** + +NameNodeMXBean#getNNStarted() metric is deprecated in branch-2 and removed from trunk. + + +--- + +* [HADOOP-12437](https://issues.apache.org/jira/browse/HADOOP-12437) | *Major* | **Allow SecurityUtil to lookup alternate hostnames** + +HADOOP-12437 introduces two new configuration settings: hadoop.security.dns.interface and hadoop.security.dns.nameserver. These settings can be used to control how Hadoop service instances look up their own hostname and may be required in some multi-homed environments where hosts are configured with multiple hostnames in DNS or hosts files. They supersede the existing settings dfs.datanode.dns.interface and dfs.datanode.dns.nameserver. + + +--- + +* [HADOOP-12446](https://issues.apache.org/jira/browse/HADOOP-12446) | *Major* | **Undeprecate createNonRecursive()** + +FileSystem#createNonRecursive() is undeprecated. + + +--- + +* [HDFS-8696](https://issues.apache.org/jira/browse/HDFS-8696) | *Major* | **Make the lower and higher watermark in the DN Netty server configurable** + +Introduced two new configuration dfs.webhdfs.netty.low.watermark and dfs.webhdfs.netty.high.watermark to enable tuning the size of the buffers of the Netty server inside Datanodes. + + +--- + +* [HDFS-9184](https://issues.apache.org/jira/browse/HDFS-9184) | *Major* | **Logging HDFS operation's caller context into audit logs** + +The feature needs to enabled by setting "hadoop.caller.context.enabled" to true. When the feature is used, additional fields are written into namenode audit log records. + + +--- + +* [HDFS-9259](https://issues.apache.org/jira/browse/HDFS-9259) | *Major* | **Make SO\_SNDBUF size configurable at DFSClient side for hdfs write scenario** + +Introduces a new configuration setting dfs.client.socket.send.buffer.size to control the socket send buffer size for writes. Setting it to zero enables TCP auto-tuning on systems that support it. + + +--- + +* [HDFS-9311](https://issues.apache.org/jira/browse/HDFS-9311) | *Major* | **Support optional offload of NameNode HA service health checks to a separate RPC server.** + +There is now support for offloading HA health check RPC activity to a separate RPC server endpoint running within the NameNode process. This may improve reliability of HA health checks and prevent spurious failovers in highly overloaded conditions. For more details, please refer to the hdfs-default.xml documentation for properties dfs.namenode.lifeline.rpc-address, dfs.namenode.lifeline.rpc-bind-host and dfs.namenode.lifeline.handler.count. + + +--- + +* [HDFS-6200](https://issues.apache.org/jira/browse/HDFS-6200) | *Major* | **Create a separate jar for hdfs-client** + +Projects that access HDFS can depend on the hadoop-hdfs-client module instead of the hadoop-hdfs module to avoid pulling in unnecessary dependency. +Please note that hadoop-hdfs-client module could miss class like ConfiguredFailoverProxyProvider. So if a cluster is in HA deployment, we should still use hadoop-hdfs instead. + + +--- + +* [HDFS-9057](https://issues.apache.org/jira/browse/HDFS-9057) | *Major* | **allow/disallow snapshots via webhdfs** + +Snapshots can be allowed/disallowed on a directory via WebHdfs from users with superuser privilege. + + +--- + +* [MAPREDUCE-5485](https://issues.apache.org/jira/browse/MAPREDUCE-5485) | *Critical* | **Allow repeating job commit by extending OutputCommitter API** + +Previously, the MR job will get failed if AM get restarted for some reason (like node failure, etc.) during its doing commit job no matter if AM attempts reach to the maximum attempts. +In this improvement, we add a new API isCommitJobRepeatable() to OutputCommitter interface which to indicate if job's committer can do commitJob again if previous commit work is interrupted by NM/AM failures, etc. The instance of OutputCommitter, which support repeatable job commit (like FileOutputCommitter in algorithm 2), can allow AM to continue the commitJob() after AM restart as a new attempt. + + +--- + +* [HADOOP-12313](https://issues.apache.org/jira/browse/HADOOP-12313) | *Critical* | **NPE in JvmPauseMonitor when calling stop() before start()** + +Allow stop() before start() completed in JvmPauseMonitor + + +--- + +* [HDFS-9433](https://issues.apache.org/jira/browse/HDFS-9433) | *Major* | **DFS getEZForPath API on a non-existent file should throw FileNotFoundException** + +Unify the behavior of dfs.getEZForPath() API when getting a non-existent normal file and non-existent ezone file by throwing FileNotFoundException + + +--- + +* [HDFS-8335](https://issues.apache.org/jira/browse/HDFS-8335) | *Major* | **FSNamesystem should construct FSPermissionChecker only if permission is enabled** + +Only check permissions when permissions enabled in FSDirStatAndListingOp.getFileInfo() and getListingInt() + + +--- + +* [HDFS-8831](https://issues.apache.org/jira/browse/HDFS-8831) | *Major* | **Trash Support for deletion in HDFS encryption zone** + +Add Trash support for deleting files within encryption zones. Deleted files will remain encrypted and they will be moved to a “.Trash” subdirectory under the root of the encryption zone, prefixed by $USER/current. Checkpoint and expunge continue to work like the existing Trash. + + +--- + +* [HDFS-9214](https://issues.apache.org/jira/browse/HDFS-9214) | *Major* | **Support reconfiguring dfs.datanode.balance.max.concurrent.moves without DN restart** + +Steps to reconfigure: +1. change value of the parameter in corresponding xml configuration file +2. to reconfigure, run + hdfs dfsadmin -reconfig datanode \:\ start +3. repeat step 2 until all DNs are reconfigured +4. to check status of the most recent reconfigure operation, run + hdfs dfsadmin -reconfig datanode \:\ status +5. to query a list reconfigurable properties on DN, run + hdfs dfsadmin -reconfig datanode \:\ properties + + +--- + +* [YARN-3623](https://issues.apache.org/jira/browse/YARN-3623) | *Major* | **We should have a config to indicate the Timeline Service version** + +Add a new configuration "yarn.timeline-service.version" to indicate what is the current version of the running timeline service. For example, if "yarn.timeline-service.version" is 1.5, and "yarn.timeline-service.enabled" is true, it means the cluster will and should bring up the timeline service v.1.5. On the client side, if the client uses the same version of timeline service, it should succeed. If the client chooses to use a smaller version in spite of this, then depending on how robust the compatibility story is between versions, the results may vary. + + +--- + +* [YARN-4207](https://issues.apache.org/jira/browse/YARN-4207) | *Major* | **Add a non-judgemental YARN app completion status** + +Adds the ENDED attribute to o.a.h.yarn.api.records.FinalApplicationStatus + + +--- + +* [HADOOP-12657](https://issues.apache.org/jira/browse/HADOOP-12657) | *Minor* | **Add a option to skip newline on empty files with getMerge -nl** + +Added -skip-empty-file option to hadoop fs -getmerge command. With the option, delimiter (LF) is not printed for empty files even if -nl option is used. + + +--- + +* [HADOOP-11252](https://issues.apache.org/jira/browse/HADOOP-11252) | *Critical* | **RPC client does not time out by default** + +This fix includes public method interface change. +A follow-up jira for this incompatibly for branch-2.7 is HADOOP-13579. + + +--- + +* [HDFS-9047](https://issues.apache.org/jira/browse/HDFS-9047) | *Major* | **Retire libwebhdfs** + +libwebhdfs has been retired in 2.8.0 due to the lack of maintenance. + + +--- + +* [HADOOP-11262](https://issues.apache.org/jira/browse/HADOOP-11262) | *Major* | **Enable YARN to use S3A** + +S3A has been made accessible through the FileContext API. + + +--- + +* [HADOOP-12635](https://issues.apache.org/jira/browse/HADOOP-12635) | *Major* | **Adding Append API support for WASB** + +The Azure Blob Storage file system (WASB) now includes optional support for use of the append API by a single writer on a path. Please note that the implementation differs from the semantics of HDFS append. HDFS append internally guarantees that only a single writer may append to a path at a given time. WASB does not enforce this guarantee internally. Instead, the application must enforce access by a single writer, such as by running single-threaded or relying on some external locking mechanism to coordinate concurrent processes. Refer to the Azure Blob Storage documentation page for more details on enabling append in configuration. + + +--- + +* [HADOOP-12651](https://issues.apache.org/jira/browse/HADOOP-12651) | *Major* | **Replace dev-support with wrappers to Yetus** + + + +* Major portions of dev-support have been replaced with wrappers to Apache Yetus: + * releasedocmaker.py is now dev-support/bin/releasedocmaker + * shelldocs.py is now dev-support/bin/shelldocs + * smart-apply-patch.sh is now dev-support/bin/smart-apply-patch + * test-patch.sh is now dev-support/bin/test-patch +* See the dev-support/README.md file for more details on how to control the wrappers to various degrees. + + +--- + +* [HDFS-9503](https://issues.apache.org/jira/browse/HDFS-9503) | *Major* | **Replace -namenode option with -fs for NNThroughputBenchmark** + +The patch replaces -namenode option with -fs for specifying the remote name node against which the benchmark is running. Before this patch, if '-namenode' was not given, the benchmark would run in standalone mode, ignoring the 'fs.defaultFS' in config file even if it's remote. With this patch, the benchmark, as other tools, will rely on the 'fs.defaultFS' config, which is overridable by -fs command option, to run standalone mode or remote mode. + + +--- + +* [HADOOP-12426](https://issues.apache.org/jira/browse/HADOOP-12426) | *Minor* | **Add Entry point for Kerberos health check** + +Hadoop now includes a shell command named KDiag that helps with diagnosis of Kerberos misconfiguration problems. Please refer to the Secure Mode documentation for full details on usage of the command. + + +--- + +* [HADOOP-12805](https://issues.apache.org/jira/browse/HADOOP-12805) | *Major* | **Annotate CanUnbuffer with @InterfaceAudience.Public** + +Made CanBuffer interface public for use in client applications. + + +--- + +* [HADOOP-12548](https://issues.apache.org/jira/browse/HADOOP-12548) | *Major* | **Read s3a creds from a Credential Provider** + +The S3A Hadoop-compatible file system now support reading its S3 credentials from the Hadoop Credential Provider API in addition to XML configuration files. + + +--- + +* [HDFS-9711](https://issues.apache.org/jira/browse/HDFS-9711) | *Major* | **Integrate CSRF prevention filter in WebHDFS.** + +WebHDFS now supports options to enforce cross-site request forgery (CSRF) prevention for HTTP requests to both the NameNode and the DataNode. Please refer to the updated WebHDFS documentation for a description of this feature and further details on how to configure it. + + +--- + +* [HADOOP-12794](https://issues.apache.org/jira/browse/HADOOP-12794) | *Major* | **Support additional compression levels for GzipCodec** + +Added New compression levels for GzipCodec that can be set in zlib.compress.level + + +--- + +* [HDFS-9425](https://issues.apache.org/jira/browse/HDFS-9425) | *Major* | **Expose number of blocks per volume as a metric** + +Number of blocks per volume is made available as a metric. + + +--- + +* [HADOOP-12668](https://issues.apache.org/jira/browse/HADOOP-12668) | *Critical* | **Support excluding weak Ciphers in HttpServer2 through ssl-server.conf** + +The Code Changes include following: +- Modified DFSUtil.java in Apache HDFS project for supplying new parameter ssl.server.exclude.cipher.list +- Modified HttpServer2.java in Apache Hadoop-common project to work with new parameter and exclude ciphers using jetty setExcludeCihers method. +- Modfied associated test classes to owrk with existing code and also cover the newfunctionality in junit + + +--- + +* [HADOOP-12555](https://issues.apache.org/jira/browse/HADOOP-12555) | *Minor* | **WASB to read credentials from a credential provider** + +The hadoop-azure file system now supports configuration of the Azure Storage account credentials using the standard Hadoop Credential Provider API. For details, please refer to the documentation on hadoop-azure and the Credential Provider API. + + +--- + +* [MAPREDUCE-6622](https://issues.apache.org/jira/browse/MAPREDUCE-6622) | *Critical* | **Add capability to set JHS job cache to a task-based limit** + +Two recommendations for the mapreduce.jobhistory.loadedtasks.cache.size property: +1) For every 100k of cache size, set the heap size of the Job History Server to 1.2GB. For example, mapreduce.jobhistory.loadedtasks.cache.size=500000, heap size=6GB. +2) Make sure that the cache size is larger than the number of tasks required for the largest job run on the cluster. It might be a good idea to set the value slightly higher (say, 20%) in order to allow for job size growth. + + +--- + +* [HADOOP-12552](https://issues.apache.org/jira/browse/HADOOP-12552) | *Minor* | **Fix undeclared/unused dependency to httpclient** + +Dependency on commons-httpclient::commons-httpclient was removed from hadoop-common. Downstream projects using commons-httpclient transitively provided by hadoop-common need to add explicit dependency to their pom. Since commons-httpclient is EOL, it is recommended to migrate to org.apache.httpcomponents:httpclient which is the successor. + + +--- + +* [HDFS-8791](https://issues.apache.org/jira/browse/HDFS-8791) | *Blocker* | **block ID-based DN storage layout can be very slow for datanode on ext4** + +HDFS-8791 introduces a new datanode layout format. This layout is identical to the previous block id based layout except it has a smaller 32x32 sub-directory structure in each data storage. On startup, the datanode will automatically upgrade it's storages to this new layout. Currently, datanode layout changes support rolling upgrades, on the other hand downgrading is not supported between datanode layout changes and a rollback would be required. + + +--- + +* [HDFS-9887](https://issues.apache.org/jira/browse/HDFS-9887) | *Major* | **WebHdfs socket timeouts should be configurable** + +Added new configuration options: dfs.webhdfs.socket.connect-timeout and dfs.webhdfs.socket.read-timeout both defaulting to 60s. + + +--- + +* [HADOOP-11792](https://issues.apache.org/jira/browse/HADOOP-11792) | *Major* | **Remove all of the CHANGES.txt files** + +With the introduction of the markdown-formatted and automatically built changes file, the CHANGES.txt files have been eliminated. + + +--- + +* [HDFS-9239](https://issues.apache.org/jira/browse/HDFS-9239) | *Major* | **DataNode Lifeline Protocol: an alternative protocol for reporting DataNode liveness** + +This release adds a new feature called the DataNode Lifeline Protocol. If configured, then DataNodes can report that they are still alive to the NameNode via a fallback protocol, separate from the existing heartbeat messages. This can prevent the NameNode from incorrectly marking DataNodes as stale or dead in highly overloaded clusters where heartbeat processing is suffering delays. For more information, please refer to the hdfs-default.xml documentation for several new configuration properties: dfs.namenode.lifeline.rpc-address, dfs.namenode.lifeline.rpc-bind-host, dfs.datanode.lifeline.interval.seconds, dfs.namenode.lifeline.handler.ratio and dfs.namenode.lifeline.handler.count. + + +--- + +* [YARN-4785](https://issues.apache.org/jira/browse/YARN-4785) | *Major* | **inconsistent value type of the "type" field for LeafQueueInfo in response of RM REST API - cluster/scheduler** + +Fix inconsistent value type ( String and Array ) of the "type" field for LeafQueueInfo in response of RM REST API + + +--- + +* [MAPREDUCE-6670](https://issues.apache.org/jira/browse/MAPREDUCE-6670) | *Minor* | **TestJobListCache#testEviction sometimes fails on Windows with timeout** + +Backport the fix to 2.7 and 2.8 + + +--- + +* [HDFS-9945](https://issues.apache.org/jira/browse/HDFS-9945) | *Major* | **Datanode command for evicting writers** + +This new dfsadmin command, evictWriters, stops active block writing activities on a data node. The affected writes will continue without the node after a write pipeline recovery. This is useful when data node decommissioning is blocked by slow writers. If issued against a non-decommissioing data node, all current writers will be stopped, but new write requests will continue to be served. + + +--- + +* [HADOOP-12963](https://issues.apache.org/jira/browse/HADOOP-12963) | *Minor* | **Allow using path style addressing for accessing the s3 endpoint** + +Add new flag to allow supporting path style addressing for s3a + + +--- + +* [HDFS-3702](https://issues.apache.org/jira/browse/HDFS-3702) | *Minor* | **Add an option for NOT writing the blocks locally if there is a datanode on the same box as the client** + +This patch will attempt to allocate all replicas to remote DataNodes, by adding local DataNode to the excluded DataNodes. If no sufficient replicas can be obtained, it will fall back to default block placement policy, which writes one replica to local DataNode. + + +--- + +* [HDFS-9902](https://issues.apache.org/jira/browse/HDFS-9902) | *Major* | **Support different values of dfs.datanode.du.reserved per storage type** + +Reserved space can be configured independently for different storage types for clusters with heterogeneous storage. The 'dfs.datanode.du.reserved' property name can be suffixed with a storage types (i.e. one of ssd, disk, archival or ram\_disk). e.g. reserved space for RAM\_DISK storage can be configured using the property 'dfs.datanode.du.reserved.ram\_disk'. If specific storage type reservation is not configured then the value specified by 'dfs.datanode.du.reserved' will be used for all volumes. + + +--- + +* [HDFS-10324](https://issues.apache.org/jira/browse/HDFS-10324) | *Major* | **Trash directory in an encryption zone should be pre-created with correct permissions** + +HDFS will create a ".Trash" subdirectory when creating a new encryption zone to support soft delete for files deleted within the encryption zone. A new "crypto -provisionTrash" command has been introduced to provision trash directories for encryption zones created with Apache Hadoop minor releases prior to 2.8.0. + + +--- + +* [HADOOP-13122](https://issues.apache.org/jira/browse/HADOOP-13122) | *Minor* | **Customize User-Agent header sent in HTTP requests by S3A.** + +S3A now includes the current Hadoop version in the User-Agent string passed through the AWS SDK to the S3 service. Users also may include optional additional information to identify their application. See the documentation of configuration property fs.s3a.user.agent.prefix for further details. + + +--- + +* [HADOOP-12723](https://issues.apache.org/jira/browse/HADOOP-12723) | *Major* | **S3A: Add ability to plug in any AWSCredentialsProvider** + +Users can integrate a custom credential provider with S3A. See documentation of configuration property fs.s3a.aws.credentials.provider for further details. + + +--- + +* [MAPREDUCE-6607](https://issues.apache.org/jira/browse/MAPREDUCE-6607) | *Minor* | **Enable regex pattern matching when mapreduce.task.files.preserve.filepattern is set** + +Before this fix, the files in .staging directory are always preserved when mapreduce.task.files.preserve.filepattern is set. After this fix, the files in .staging directory are preserved if the name of the directory matches the regex pattern specified by mapreduce.task.files.preserve.filepattern. + + +--- + +* [YARN-5035](https://issues.apache.org/jira/browse/YARN-5035) | *Major* | **FairScheduler: Adjust maxAssign dynamically when assignMultiple is turned on** + +Introducing a new configuration "yarn.scheduler.fair.dynamic.max.assign" to dynamically determine the resources to assign per heartbeat when assignmultiple is turned on. When turned on, the scheduler allocates roughly half of the remaining resources overriding any max.assign settings configured. This is turned ON by default. + + +--- + +* [YARN-5132](https://issues.apache.org/jira/browse/YARN-5132) | *Critical* | **Exclude generated protobuf sources from YARN Javadoc build** + +Exclude javadocs for proto-generated java classes. + + +--- + +* [HADOOP-13105](https://issues.apache.org/jira/browse/HADOOP-13105) | *Major* | **Support timeouts in LDAP queries in LdapGroupsMapping.** + +This patch adds two new config keys for supporting timeouts in LDAP query operations. The property "hadoop.security.group.mapping.ldap.connection.timeout.ms" is the connection timeout (in milliseconds), within which period if the LDAP provider doesn't establish a connection, it will abort the connect attempt. The property "hadoop.security.group.mapping.ldap.read.timeout.ms" is the read timeout (in milliseconds), within which period if the LDAP provider doesn't get a LDAP response, it will abort the read attempt. + + +--- + +* [HADOOP-13155](https://issues.apache.org/jira/browse/HADOOP-13155) | *Major* | **Implement TokenRenewer to renew and cancel delegation tokens in KMS** + +Enables renewal and cancellation of KMS delegation tokens. hadoop.security.key.provider.path needs to be configured to reach the key provider. + + +--- + +* [HADOOP-12807](https://issues.apache.org/jira/browse/HADOOP-12807) | *Minor* | **S3AFileSystem should read AWS credentials from environment variables** + +Adds support to S3AFileSystem for reading AWS credentials from environment variables. + + +--- + +* [HDFS-10375](https://issues.apache.org/jira/browse/HDFS-10375) | *Trivial* | **Remove redundant TestMiniDFSCluster.testDualClusters** + +Remove redundent TestMiniDFSCluster.testDualClusters to save time. + + +--- + +* [HDFS-10220](https://issues.apache.org/jira/browse/HDFS-10220) | *Major* | **A large number of expired leases can make namenode unresponsive and cause failover** + +Two new configuration have been added "dfs.namenode.lease-recheck-interval-ms" and "dfs.namenode.max-lock-hold-to-release-lease-ms" to fine tune the duty cycle with which the Namenode recovers old leases. + + +--- + +* [HADOOP-13237](https://issues.apache.org/jira/browse/HADOOP-13237) | *Minor* | **s3a initialization against public bucket fails if caller lacks any credentials** + +S3A now supports read access to a public S3 bucket even if the client does not configure any AWS credentials. See the documentation of configuration property fs.s3a.aws.credentials.provider for further details. + + +--- + +* [HADOOP-12537](https://issues.apache.org/jira/browse/HADOOP-12537) | *Minor* | **S3A to support Amazon STS temporary credentials** + +S3A now supports use of AWS Security Token Service temporary credentials for authentication to S3. Refer to the documentation of configuration property fs.s3a.session.token for further details. + + +--- + +* [HADOOP-12892](https://issues.apache.org/jira/browse/HADOOP-12892) | *Blocker* | **fix/rewrite create-release** + +This rewrites the release process with a new dev-support/bin/create-release script. See http://wiki.apache.org/hadoop/HowToRelease for updated instructions on how to use it. + + +--- + +* [HADOOP-3733](https://issues.apache.org/jira/browse/HADOOP-3733) | *Minor* | **"s3:" URLs break when Secret Key contains a slash, even if encoded** + +Allows userinfo component of URI authority to contain a slash (escaped as %2F). Especially useful for accessing AWS S3 with distcp or hadoop fs. + + +--- + +* [HADOOP-13203](https://issues.apache.org/jira/browse/HADOOP-13203) | *Major* | **S3A: Support fadvise "random" mode for high performance readPositioned() reads** + +S3A has added support for configurable input policies. Similar to fadvise, this configuration provides applications with a way to specify their expected access pattern (sequential or random) while reading a file. S3A then performs optimizations tailored to that access pattern. See site documentation of the fs.s3a.experimental.input.fadvise configuration property for more details. Please be advised that this feature is experimental and subject to backward-incompatible changes in future releases. + + +--- + +* [HADOOP-13263](https://issues.apache.org/jira/browse/HADOOP-13263) | *Major* | **Reload cached groups in background after expiry** + +hadoop.security.groups.cache.background.reload can be set to true to enable background reload of expired groups cache entries. This setting can improve the performance of services that use Groups.java (e.g. the NameNode) when group lookups are slow. The setting is disabled by default. + + +--- + +* [HDFS-10440](https://issues.apache.org/jira/browse/HDFS-10440) | *Major* | **Improve DataNode web UI** + +DataNode Web UI has been improved with new HTML5 page, showing useful information. + + +--- + +* [HADOOP-13139](https://issues.apache.org/jira/browse/HADOOP-13139) | *Major* | **Branch-2: S3a to use thread pool that blocks clients** + +The configuration option 'fs.s3a.threads.core' is no longer supported. The string is still defined in org.apache.hadoop.fs.s3a.Constants.CORE\_THREADS, however its value is ignored. If it is set, a warning message will be printed when initializing the S3A filesystem + + +--- + +* [HADOOP-13382](https://issues.apache.org/jira/browse/HADOOP-13382) | *Major* | **remove unneeded commons-httpclient dependencies from POM files in Hadoop and sub-projects** + +Dependencies on commons-httpclient have been removed. Projects with undeclared transitive dependencies on commons-httpclient, previously provided via hadoop-common or hadoop-client, may find this to be an incompatible change. Such project are also potentially exposed to the commons-httpclient CVE, and should be fixed for that reason as well. + + +--- + +* [HDFS-7933](https://issues.apache.org/jira/browse/HDFS-7933) | *Major* | **fsck should also report decommissioning replicas.** + +The output of hdfs fsck now also contains information about decommissioning replicas. + + +--- + +* [HADOOP-13208](https://issues.apache.org/jira/browse/HADOOP-13208) | *Minor* | **S3A listFiles(recursive=true) to do a bulk listObjects instead of walking the pseudo-tree of directories** + +S3A has optimized the listFiles method by doing a bulk listing of all entries under a path in a single S3 operation instead of recursively walking the directory tree. The listLocatedStatus method has been optimized by fetching results from S3 lazily as the caller traverses the returned iterator instead of doing an eager fetch of all possible results. + + +--- + +* [HADOOP-13252](https://issues.apache.org/jira/browse/HADOOP-13252) | *Minor* | **Tune S3A provider plugin mechanism** + +S3A now supports configuration of multiple credential provider classes for authenticating to S3. These are loaded and queried in sequence for a valid set of credentials. For more details, refer to the description of the fs.s3a.aws.credentials.provider configuration property or the S3A documentation page. + + +--- + +* [HDFS-8986](https://issues.apache.org/jira/browse/HDFS-8986) | *Major* | **Add option to -du to calculate directory space usage excluding snapshots** + +Add a -x option for "hdfs -du" and "hdfs -count" commands to exclude snapshots from being calculated. + + +--- + +* [HDFS-10760](https://issues.apache.org/jira/browse/HDFS-10760) | *Major* | **DataXceiver#run() should not log InvalidToken exception as an error** + +Log InvalidTokenException at trace level in DataXceiver#run(). + + +--- + +* [YARN-5549](https://issues.apache.org/jira/browse/YARN-5549) | *Critical* | **AMLauncher#createAMContainerLaunchContext() should not log the command to be launched indiscriminately** + +Introduces a new configuration property, yarn.resourcemanager.amlauncher.log.command. If this property is set to true, then the AM command being launched will be masked in the RM log. + + +--- + +* [HDFS-8818](https://issues.apache.org/jira/browse/HDFS-8818) | *Major* | **Allow Balancer to run faster** + +Add a new conf "dfs.balancer.max-size-to-move" so that Balancer.MAX\_SIZE\_TO\_MOVE becomes configurable. + + +--- + +* [HDFS-10489](https://issues.apache.org/jira/browse/HDFS-10489) | *Minor* | **Deprecate dfs.encryption.key.provider.uri for HDFS encryption zones** + +The configuration dfs.encryption.key.provider.uri is deprecated. To configure key provider in HDFS, please use hadoop.security.key.provider.path. + + +--- + +* [HDFS-10914](https://issues.apache.org/jira/browse/HDFS-10914) | *Critical* | **Move remnants of oah.hdfs.client to hadoop-hdfs-client** + +The remaining classes in the org.apache.hadoop.hdfs.client package have been moved from hadoop-hdfs to hadoop-hdfs-client. + + +--- + +* [HADOOP-12667](https://issues.apache.org/jira/browse/HADOOP-12667) | *Major* | **s3a: Support createNonRecursive API** + +S3A now provides a working implementation of the FileSystem#createNonRecursive method. + + +--- + +* [HDFS-10609](https://issues.apache.org/jira/browse/HDFS-10609) | *Major* | **Uncaught InvalidEncryptionKeyException during pipeline recovery may abort downstream applications** + +If pipeline recovery fails due to expired encryption key, attempt to refresh the key and retry. + + +--- + +* [HDFS-10797](https://issues.apache.org/jira/browse/HDFS-10797) | *Major* | **Disk usage summary of snapshots causes renamed blocks to get counted twice** + +Disk usage summaries previously incorrectly counted files twice if they had been renamed (including files moved to Trash) since being snapshotted. Summaries now include current data plus snapshotted data that is no longer under the directory either due to deletion or being moved outside of the directory. + + +--- + +* [HDFS-10883](https://issues.apache.org/jira/browse/HDFS-10883) | *Major* | **`getTrashRoot`'s behavior is not consistent in DFS after enabling EZ.** + +If root path / is an encryption zone, the old DistributedFileSystem#getTrashRoot(new Path("/")) returns +/user/$USER/.Trash +which is a wrong behavior. The correct value should be +/.Trash/$USER + + +--- + +* [HADOOP-13560](https://issues.apache.org/jira/browse/HADOOP-13560) | *Major* | **S3ABlockOutputStream to support huge (many GB) file writes** + +This mechanism replaces the (experimental) fast output stream of Hadoop 2.7.x, combining better scalability options with instrumentation. Consult the S3A documentation to see the extra configuration operations. + + +--- + +* [HDFS-11018](https://issues.apache.org/jira/browse/HDFS-11018) | *Major* | **Incorrect check and message in FsDatasetImpl#invalidate** + +Improves the error message when datanode removes a replica which is not found. + + +--- + +* [YARN-5767](https://issues.apache.org/jira/browse/YARN-5767) | *Major* | **Fix the order that resources are cleaned up from the local Public/Private caches** + +This issue fixes a bug in how resources are evicted from the PUBLIC and PRIVATE yarn local caches used by the node manager for resource localization. In summary, the caches are now properly cleaned based on an LRU policy across both the public and private caches. + + +--- + +* [HDFS-11048](https://issues.apache.org/jira/browse/HDFS-11048) | *Major* | **Audit Log should escape control characters** + +HDFS audit logs are formatted as individual lines, each of which has a few of key-value pair fields. Some of the values come from client request (e.g. src, dst). Before this patch the control characters including \t \n etc are not escaped in audit logs. That may break lines unexpectedly or introduce additional table character (in the worst case, both) within a field. Tools that parse audit logs had to deal with this case carefully. After this patch, the control characters in the src/dst fields are escaped. + + +--- + +* [HADOOP-10597](https://issues.apache.org/jira/browse/HADOOP-10597) | *Major* | **RPC Server signals backoff to clients when all request queues are full** + +This change introduces a new configuration key used by RPC server to decide whether to send backoff signal to RPC Client when RPC call queue is full. When the feature is enabled, RPC server will no longer block on the processing of RPC requests when RPC call queue is full. It helps to improve quality of service when the service is under heavy load. The configuration key is in the format of "ipc.#port#.backoff.enable" where #port# is the port number that RPC server listens on. For example, if you want to enable the feature for the RPC server that listens on 8020, set ipc.8020.backoff.enable to true. + + +--- + +* [HDFS-11056](https://issues.apache.org/jira/browse/HDFS-11056) | *Major* | **Concurrent append and read operations lead to checksum error** + +Load last partial chunk checksum properly into memory when converting a finalized/temporary replica to rbw replica. This ensures concurrent reader reads the correct checksum that matches the data before the update. + + +--- + +* [HADOOP-13812](https://issues.apache.org/jira/browse/HADOOP-13812) | *Blocker* | **Upgrade Tomcat to 6.0.48** + +Tomcat 6.0.46 starts to filter weak ciphers. Some old SSL clients may be affected. It is recommended to upgrade the SSL client. Run the SSL client against https://www.howsmyssl.com/a/check to find out its TLS version and cipher suites. + + +--- + +* [HDFS-11217](https://issues.apache.org/jira/browse/HDFS-11217) | *Major* | **Annotate NameNode and DataNode MXBean interfaces as Private/Stable** + +The DataNode and NameNode MXBean interfaces have been marked as Private and Stable to indicate that although users should not be implementing these interfaces directly, the information exposed by these interfaces is part of the HDFS public API. + + +--- + +* [HDFS-11229](https://issues.apache.org/jira/browse/HDFS-11229) | *Blocker* | **HDFS-11056 failed to close meta file** + +The fix for HDFS-11056 reads meta file to load last partial chunk checksum when a block is converted from finalized/temporary to rbw. However, it did not close the file explicitly, which may cause number of open files reaching system limit. This jira fixes it by closing the file explicitly after the meta file is read. + + +--- + +* [HDFS-11160](https://issues.apache.org/jira/browse/HDFS-11160) | *Major* | **VolumeScanner reports write-in-progress replicas as corrupt incorrectly** + +Fixed a race condition that caused VolumeScanner to recognize a good replica as a bad one if the replica is also being written concurrently. + + +--- + +* [HADOOP-13956](https://issues.apache.org/jira/browse/HADOOP-13956) | *Critical* | **Read ADLS credentials from Credential Provider** + +The hadoop-azure-datalake file system now supports configuration of the Azure Data Lake Store account credentials using the standard Hadoop Credential Provider API. For details, please refer to the documentation on hadoop-azure-datalake and the Credential Provider API. + + +--- + +* [YARN-5271](https://issues.apache.org/jira/browse/YARN-5271) | *Major* | **ATS client doesn't work with Jersey 2 on the classpath** + +A workaround to avoid dependency conflict with Spark2, before a full classpath isolation solution is implemented. +Skip instantiating a Timeline Service client if encountering NoClassDefFoundError. + + +--- + +* [HADOOP-13929](https://issues.apache.org/jira/browse/HADOOP-13929) | *Major* | **ADLS connector should not check in contract-test-options.xml** + +To run live unit tests, create src/test/resources/auth-keys.xml with the same properties as in the deprecated contract-test-options.xml. + + +--- + +* [YARN-6177](https://issues.apache.org/jira/browse/YARN-6177) | *Major* | **Yarn client should exit with an informative error message if an incompatible Jersey library is used at client** + +Let yarn client exit with an informative error message if an incompatible Jersey library is used from client side. + + +--- + +* [HDFS-11498](https://issues.apache.org/jira/browse/HDFS-11498) | *Major* | **Make RestCsrfPreventionHandler and WebHdfsHandler compatible with Netty 4.0** + +This JIRA sets the Netty 4 dependency to 4.0.23. This is an incompatible change for the 3.0 release line, as 3.0.0-alpha1 and 3.0.0-alpha2 depended on Netty 4.1.0.Beta5. + + +--- + +* [HADOOP-13037](https://issues.apache.org/jira/browse/HADOOP-13037) | *Major* | **Refactor Azure Data Lake Store as an independent FileSystem** + +Hadoop now supports integration with Azure Data Lake as an alternative Hadoop-compatible file system. Please refer to the Hadoop site documentation of Azure Data Lake for details on usage and configuration. + + +--- + +* [HDFS-11431](https://issues.apache.org/jira/browse/HDFS-11431) | *Blocker* | **hadoop-hdfs-client JAR does not include ConfiguredFailoverProxyProvider** + +The hadoop-client POM now includes a leaner hdfs-client, stripping out all the transitive dependencies on JARs only needed for the Hadoop HDFS daemon itself. The specific jars now excluded are: leveldbjni-all, jetty-util, commons-daemon, xercesImpl, netty and servlet-api. + +This should make downstream projects dependent JARs smaller, and avoid version conflict problems with the specific JARs now excluded. + +Applications may encounter build problems if they did depend on these JARs, and which didn't explicitly include them. There are two fixes for this + +\* explicitly include the JARs, stating which version of them you want. +\* add a dependency on hadoop-hdfs. For Hadoop 2.8+, this will add the missing dependencies. For builds against older versions of Hadoop, this will be harmless, as hadoop-hdfs and all its dependencies are already pulled in by the hadoop-client POM. + + + From d174c06b01e1f743d3111b9b760a9824d8106b86 Mon Sep 17 00:00:00 2001 From: Junping Du Date: Sat, 25 Mar 2017 14:05:20 -0700 Subject: [PATCH 132/188] Add HDFS jdiff for 2.8.0 release. --- .../jdiff/Apache_Hadoop_HDFS_2.8.0.xml | 312 ++++++++++++++++++ 1 file changed, 312 insertions(+) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_2.8.0.xml diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_2.8.0.xml b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_2.8.0.xml new file mode 100644 index 00000000000..0428c638257 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_2.8.0.xml @@ -0,0 +1,312 @@ + + + + + + + + + + + A distributed implementation of {@link +org.apache.hadoop.fs.FileSystem}. This is loosely modelled after +Google's GFS.

      + +

      The most important difference is that unlike GFS, Hadoop DFS files +have strictly one writer at any one time. Bytes are always appended +to the end of the writer's stream. There is no notion of "record appends" +or "mutations" that are then checked or reordered. Writers simply emit +a byte stream. That byte stream is guaranteed to be stored in the +order written.

      ]]> +       +       + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This method must return as quickly as possible, since it's called + in a critical section of the NameNode's operation. + + @param succeeded Whether authorization succeeded. + @param userName Name of the user executing the request. + @param addr Remote address of the request. + @param cmd The requested command. + @param src Path of affected source file. + @param dst Path of affected destination file (if any). + @param stat File information for operations that change the file's + metadata (permissions, owner, times, etc).]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +       From 6df029db36412b7219b64313dcbe1874dc1c8b0c Mon Sep 17 00:00:00 2001 From: Junping Du Date: Sat, 25 Mar 2017 14:28:48 -0700 Subject: [PATCH 133/188] Add COMMON/MAPREDUCE/YARN jdiff of 2.8.0 release. --- .../jdiff/Apache_Hadoop_Common_2.8.0.xml | 37921 ++++++++++++++++ .../Apache_Hadoop_MapReduce_Common_2.8.0.xml | 113 + .../Apache_Hadoop_MapReduce_Core_2.8.0.xml | 27490 +++++++++++ ...pache_Hadoop_MapReduce_JobClient_2.8.0.xml | 16 + .../jdiff/Apache_Hadoop_YARN_Client_2.8.0.xml | 2316 + .../jdiff/Apache_Hadoop_YARN_Common_2.8.0.xml | 2665 ++ ...Apache_Hadoop_YARN_Server_Common_2.8.0.xml | 829 + 7 files changed, 71350 insertions(+) create mode 100644 hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_2.8.0.xml create mode 100644 hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Common_2.8.0.xml create mode 100644 hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Core_2.8.0.xml create mode 100644 hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_JobClient_2.8.0.xml create mode 100644 hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Client_2.8.0.xml create mode 100644 hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Common_2.8.0.xml create mode 100644 hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Server_Common_2.8.0.xml diff --git a/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_2.8.0.xml b/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_2.8.0.xml new file mode 100644 index 00000000000..807a1622103 --- /dev/null +++ b/hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_2.8.0.xml @@ -0,0 +1,37921 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + UnsupportedOperationException + + If a key is deprecated in favor of multiple keys, they are all treated as + aliases of each other, and setting any one of them resets all the others + to the new value. + + If you have multiple deprecation entries to add, it is more efficient to + use #addDeprecations(DeprecationDelta[] deltas) instead. + + @param key + @param newKeys + @param customMessage + @deprecated use {@link #addDeprecation(String key, String newKey, + String customMessage)} instead]]> + + + + + + + + UnsupportedOperationException + + If you have multiple deprecation entries to add, it is more efficient to + use #addDeprecations(DeprecationDelta[] deltas) instead. + + @param key + @param newKey + @param customMessage]]> + + + + + + + UnsupportedOperationException + + If a key is deprecated in favor of multiple keys, they are all treated as + aliases of each other, and setting any one of them resets all the others + to the new value. + + If you have multiple deprecation entries to add, it is more efficient to + use #addDeprecations(DeprecationDelta[] deltas) instead. + + @param key Key that is to be deprecated + @param newKeys list of keys that take up the values of deprecated key + @deprecated use {@link #addDeprecation(String key, String newKey)} instead]]> + + + + + + + UnsupportedOperationException + + If you have multiple deprecation entries to add, it is more efficient to + use #addDeprecations(DeprecationDelta[] deltas) instead. + + @param key Key that is to be deprecated + @param newKey key that takes up the value of deprecated key]]> + + + + + + key is deprecated. + + @param key the parameter which is to be checked for deprecation + @return true if the key is deprecated and + false otherwise.]]> + + + + + + + + + + + + + + + + + final. + + @param name resource to be added, the classpath is examined for a file + with that name.]]> + + + + + + final. + + @param url url of the resource to be added, the local filesystem is + examined directly to find the resource, without referring to + the classpath.]]> + + + + + + final. + + @param file file-path of resource to be added, the local filesystem is + examined directly to find the resource, without referring to + the classpath.]]> + + + + + + final. + + WARNING: The contents of the InputStream will be cached, by this method. + So use this sparingly because it does increase the memory consumption. + + @param in InputStream to deserialize the object from. In will be read from + when a get or set is called next. After it is read the stream will be + closed.]]> + + + + + + + final. + + @param in InputStream to deserialize the object from. + @param name the name of the resource because InputStream.toString is not + very descriptive some times.]]> + + + + + + final. + + @param conf Configuration object from which to load properties]]> + + + + + + + + + + + name property, null if + no such property exists. If the key is deprecated, it returns the value of + the first key which replaces the deprecated key and is not null. + + Values are processed for variable expansion + before being returned. + + @param name the property name, will be trimmed before get value. + @return the value of the name or its replacing property, + or null if no such property exists.]]> + + + + + + + + + + + + name property, but only for + names which have no valid value, usually non-existent or commented + out in XML. + + @param name the property name + @return true if the property name exists without value]]> + + + + + + name property as a trimmed String, + null if no such property exists. + If the key is deprecated, it returns the value of + the first key which replaces the deprecated key and is not null + + Values are processed for variable expansion + before being returned. + + @param name the property name. + @return the value of the name or its replacing property, + or null if no such property exists.]]> + + + + + + + name property as a trimmed String, + defaultValue if no such property exists. + See @{Configuration#getTrimmed} for more details. + + @param name the property name. + @param defaultValue the property default value. + @return the value of the name or defaultValue + if it is not set.]]> + + + + + + name property, without doing + variable expansion.If the key is + deprecated, it returns the value of the first key which replaces + the deprecated key and is not null. + + @param name the property name. + @return the value of the name property or + its replacing property and null if no such property exists.]]> + + + + + + + value of the name property. If + name is deprecated or there is a deprecated name associated to it, + it sets the value to both names. Name will be trimmed before put into + configuration. + + @param name property name. + @param value property value.]]> + + + + + + + + value of the name property. If + name is deprecated, it also sets the value to + the keys that replace the deprecated key. Name will be trimmed before put + into configuration. + + @param name property name. + @param value property value. + @param source the place that this configuration value came from + (For debugging). + @throws IllegalArgumentException when the value or name is null.]]> + + + + + + + + + + + + + + + + + + + + name. If the key is deprecated, + it returns the value of the first key which replaces the deprecated key + and is not null. + If no such property exists, + then defaultValue is returned. + + @param name property name, will be trimmed before get value. + @param defaultValue default value. + @return property value, or defaultValue if the property + doesn't exist.]]> + + + + + + + name property as an int. + + If no such property exists, the provided default value is returned, + or if the specified value is not a valid int, + then an error is thrown. + + @param name property name. + @param defaultValue default value. + @throws NumberFormatException when the value is invalid + @return property value as an int, + or defaultValue.]]> + + + + + + name property as a set of comma-delimited + int values. + + If no such property exists, an empty array is returned. + + @param name property name + @return property value interpreted as an array of comma-delimited + int values]]> + + + + + + + name property to an int. + + @param name property name. + @param value int value of the property.]]> + + + + + + + name property as a long. + If no such property exists, the provided default value is returned, + or if the specified value is not a valid long, + then an error is thrown. + + @param name property name. + @param defaultValue default value. + @throws NumberFormatException when the value is invalid + @return property value as a long, + or defaultValue.]]> + + + + + + + name property as a long or + human readable format. If no such property exists, the provided default + value is returned, or if the specified value is not a valid + long or human readable format, then an error is thrown. You + can use the following suffix (case insensitive): k(kilo), m(mega), g(giga), + t(tera), p(peta), e(exa) + + @param name property name. + @param defaultValue default value. + @throws NumberFormatException when the value is invalid + @return property value as a long, + or defaultValue.]]> + + + + + + + name property to a long. + + @param name property name. + @param value long value of the property.]]> + + + + + + + name property as a float. + If no such property exists, the provided default value is returned, + or if the specified value is not a valid float, + then an error is thrown. + + @param name property name. + @param defaultValue default value. + @throws NumberFormatException when the value is invalid + @return property value as a float, + or defaultValue.]]> + + + + + + + name property to a float. + + @param name property name. + @param value property value.]]> + + + + + + + name property as a double. + If no such property exists, the provided default value is returned, + or if the specified value is not a valid double, + then an error is thrown. + + @param name property name. + @param defaultValue default value. + @throws NumberFormatException when the value is invalid + @return property value as a double, + or defaultValue.]]> + + + + + + + name property to a double. + + @param name property name. + @param value property value.]]> + + + + + + + name property as a boolean. + If no such property is specified, or if the specified value is not a valid + boolean, then defaultValue is returned. + + @param name property name. + @param defaultValue default value. + @return property value as a boolean, + or defaultValue.]]> + + + + + + + name property to a boolean. + + @param name property name. + @param value boolean value of the property.]]> + + + + + + + + + + + + + + name property to the given type. This + is equivalent to set(<name>, value.toString()). + @param name property name + @param value new value]]> + + + + + + + + + + + + + + + name to the given time duration. This + is equivalent to set(<name>, value + <time suffix>). + @param name Property name + @param value Time duration + @param unit Unit of time]]> + + + + + + + + + + + + + + + + + + + name property as a Pattern. + If no such property is specified, or if the specified value is not a valid + Pattern, then DefaultValue is returned. + Note that the returned value is NOT trimmed by this method. + + @param name property name + @param defaultValue default value + @return property value as a compiled Pattern, or defaultValue]]> + + + + + + + Pattern. + If the pattern is passed as null, sets the empty pattern which results in + further calls to getPattern(...) returning the default value. + + @param name property name + @param pattern new value]]> + + + + + + + + + + + + + + + + + + + name property as + a collection of Strings. + If no such property is specified then empty collection is returned. +

      + This is an optimized version of {@link #getStrings(String)} + + @param name property name. + @return property value as a collection of Strings.]]> + + + + + + name property as + an array of Strings. + If no such property is specified then null is returned. + + @param name property name. + @return property value as an array of Strings, + or null.]]> + + + + + + + name property as + an array of Strings. + If no such property is specified then default value is returned. + + @param name property name. + @param defaultValue The default value + @return property value as an array of Strings, + or default value.]]> + + + + + + name property as + a collection of Strings, trimmed of the leading and trailing whitespace. + If no such property is specified then empty Collection is returned. + + @param name property name. + @return property value as a collection of Strings, or empty Collection]]> + + + + + + name property as + an array of Strings, trimmed of the leading and trailing whitespace. + If no such property is specified then an empty array is returned. + + @param name property name. + @return property value as an array of trimmed Strings, + or empty array.]]> + + + + + + + name property as + an array of Strings, trimmed of the leading and trailing whitespace. + If no such property is specified then default value is returned. + + @param name property name. + @param defaultValue The default value + @return property value as an array of trimmed Strings, + or default value.]]> + + + + + + + name property as + as comma delimited values. + + @param name property name. + @param values The values]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + hostProperty as a + InetSocketAddress. If hostProperty is + null, addressProperty will be used. This + is useful for cases where we want to differentiate between host + bind address and address clients should use to establish connection. + + @param hostProperty bind host property name. + @param addressProperty address property name. + @param defaultAddressValue the default value + @param defaultPort the default port + @return InetSocketAddress]]> + + + + + + + + name property as a + InetSocketAddress. + @param name property name. + @param defaultAddress the default value + @param defaultPort the default port + @return InetSocketAddress]]> + + + + + + + name property as + a host:port.]]> + + + + + + + + + name property as a host:port. The wildcard + address is replaced with the local host's address. If the host and address + properties are configured the host component of the address will be combined + with the port component of the addr to generate the address. This is to allow + optional control over which host name is used in multi-home bind-host + cases where a host can have multiple names + @param hostProperty the bind-host configuration name + @param addressProperty the service address configuration name + @param defaultAddressValue the service default address configuration value + @param addr InetSocketAddress of the service listener + @return InetSocketAddress for clients to connect]]> + + + + + + + name property as a host:port. The wildcard + address is replaced with the local host's address. + @param name property name. + @param addr InetSocketAddress of a listener to store in the given property + @return InetSocketAddress for clients to connect]]> + + + + + + + + + + + + + + + + + + + + name property + as an array of Class. + The value of the property specifies a list of comma separated class names. + If no such property is specified, then defaultValue is + returned. + + @param name the property name. + @param defaultValue default value. + @return property value as a Class[], + or defaultValue.]]> + + + + + + + name property as a Class. + If no such property is specified, then defaultValue is + returned. + + @param name the class name. + @param defaultValue default value. + @return property value as a Class, + or defaultValue.]]> + + + + + + + + name property as a Class + implementing the interface specified by xface. + + If no such property is specified, then defaultValue is + returned. + + An exception is thrown if the returned class does not implement the named + interface. + + @param name the class name. + @param defaultValue default value. + @param xface the interface implemented by the named class. + @return property value as a Class, + or defaultValue.]]> + + + + + + + name property as a List + of objects implementing the interface specified by xface. + + An exception is thrown if any of the classes does not exist, or if it does + not implement the named interface. + + @param name the property name. + @param xface the interface implemented by the classes named by + name. + @return a List of objects implementing xface.]]> + + + + + + + + name property to the name of a + theClass implementing the given interface xface. + + An exception is thrown if theClass does not implement the + interface xface. + + @param name property name. + @param theClass property value. + @param xface the interface implemented by the named class.]]> + + + + + + + + dirsProp with + the given path. If dirsProp contains multiple directories, + then one is chosen based on path's hash code. If the selected + directory does not exist, an attempt is made to create it. + + @param dirsProp directory in which to locate the file. + @param path file-path. + @return local file under the directory with the given path.]]> + + + + + + + + dirsProp with + the given path. If dirsProp contains multiple directories, + then one is chosen based on path's hash code. If the selected + directory does not exist, an attempt is made to create it. + + @param dirsProp directory in which to locate the file. + @param path file-path. + @return local file under the directory with the given path.]]> + + + + + + + + + + + + name. + + @param name configuration resource name. + @return an input stream attached to the resource.]]> + + + + + + name. + + @param name configuration resource name. + @return a reader attached to the resource.]]> + + + + + + + + + + + + + + + + + + + + + + String + key-value pairs in the configuration. + + @return an iterator over the entries.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true to set quiet-mode on, false + to turn it off.]]> + + + + + + + + + + + + + + + + + + + + + with matching keys]]> + + + + + + + + + + + + Resources + +

      Configurations are specified by resources. A resource contains a set of + name/value pairs as XML data. Each resource is named by either a + String or by a {@link Path}. If named by a String, + then the classpath is examined for a file with that name. If named by a + Path, then the local filesystem is examined directly, without + referring to the classpath. + +

      Unless explicitly turned off, Hadoop by default specifies two + resources, loaded in-order from the classpath:

        +
      1. + + core-default.xml: Read-only defaults for hadoop.
        2. +
      2. core-site.xml: Site-specific configuration for a given hadoop + installation.
        3. +
      + Applications may add additional resources, which are loaded + subsequent to these resources in the order they are added. + +

      Final Parameters

      + +

      Configuration parameters may be declared final. + Once a resource declares a value final, no subsequently-loaded + resource can alter that value. + For example, one might define a final parameter with: + 

      +  <property>
+    <name>dfs.hosts.include</name>
+    <value>/etc/hadoop/conf/hosts.include</value>
+    <final>true</final>
+  </property>
      + + Administrators typically define parameters as final in + core-site.xml for values that user applications may not alter. + +

      Variable Expansion

      + +

      Value strings are first processed for variable expansion. The + available properties are:

        +
      1. Other properties defined in this Configuration; and, if a name is + undefined here,
        2. +
      2. Properties in {@link System#getProperties()}.
        3. +
      + +

      For example, if a configuration resource contains the following property + definitions: + 

      +  <property>
+    <name>basedir</name>
+    <value>/user/${user.name}</value>
+  </property>
+  
+  <property>
+    <name>tempdir</name>
+    <value>${basedir}/tmp</value>
+  </property>
      + + When conf.get("tempdir") is called, then ${basedir} + will be resolved to another property in this Configuration, while + ${user.name} would then ordinarily be resolved to the value + of the System property with that name. +

      When conf.get("otherdir") is called, then ${env.BASE_DIR} + will be resolved to the value of the ${BASE_DIR} environment variable. + It supports ${env.NAME:-default} and ${env.NAME-default} notations. + The former is resolved to "default" if ${NAME} environment variable is undefined + or its value is empty. + The latter behaves the same way only if ${NAME} is undefined. +

      By default, warnings will be given to any deprecated configuration + parameters and these are suppressible by configuring + log4j.logger.org.apache.hadoop.conf.Configuration.deprecation in + log4j.properties file.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This implementation generates the key material and calls the + {@link #createKey(String, byte[], Options)} method. + + @param name the base name of the key + @param options the options for the new key. + @return the version name of the first version of the key. + @throws IOException + @throws NoSuchAlgorithmException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This implementation generates the key material and calls the + {@link #rollNewVersion(String, byte[])} method. + + @param name the basename of the key + @return the name of the new version of the key + @throws IOException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + KeyProvider implementations must be thread safe.]]> + + + + + + + + + + + + + + + + + + + + + + NULL if + a provider for the specified URI scheme could not be found. + @throws IOException thrown if the provider failed to initialize.]]> + + + + + + + + + + + + + + + + + + + + + uri has syntax error]]> + + + + + + + + + + + + + + + + uri is + not found]]> + + + + + + + + + + + + + + + + + + + + + + + uri + determines a configuration property name, + fs.AbstractFileSystem.scheme.impl whose value names the + AbstractFileSystem class. + + The entire URI and conf is passed to the AbstractFileSystem factory method. + + @param uri for the file system to be created. + @param conf which is passed to the file system impl. + + @return file system for the given URI. + + @throws UnsupportedFileSystemException if the file system for + uri is not supported.]]> + + + + + + + + + + + + default port;]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + describing modifications + @throws IOException if an ACL could not be modified]]> + + + + + + + + describing entries to remove + @throws IOException if an ACL could not be modified]]> + + + + + + + + + + + + + + + + + + + + + + describing modifications, must include entries + for user, group, and others for compatibility with permission bits. + @throws IOException if an ACL could not be modified]]> + + + + + + + which returns each AclStatus + @throws IOException if an ACL could not be read]]> + + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to modify + @param name xattr name. + @param value xattr value. + @throws IOException]]> + + + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to modify + @param name xattr name. + @param value xattr value. + @param flag xattr set flag + @throws IOException]]> + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to get extended attribute + @param name xattr name. + @return byte[] xattr value. + @throws IOException]]> + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to get extended attributes + @return Map describing the XAttrs of the file or directory + @throws IOException]]> + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to get extended attributes + @param names XAttr names. + @return Map describing the XAttrs of the file or directory + @throws IOException]]> + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to get extended attributes + @return Map describing the XAttrs of the file or directory + @throws IOException]]> + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to remove extended attribute + @param name xattr name + @throws IOException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + After a successful call, buf.position() will be advanced by the number + of bytes read and buf.limit() should be unchanged. +

      + In the case of an exception, the values of buf.position() and buf.limit() + are undefined, and callers should be prepared to recover from this + eventuality. +

      + Many implementations will throw {@link UnsupportedOperationException}, so + callers that are not confident in support for this method from the + underlying filesystem should be prepared to handle that exception. +

      + Implementations should treat 0-length requests as legitimate, and must not + signal an error upon their receipt. + + @param buf + the ByteBuffer to receive the results of the read operation. + @return the number of bytes read, possibly zero, or -1 if + reach end-of-stream + @throws IOException + if there is some error performing the read]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + setReplication of FileSystem + @param src file name + @param replication new replication + @throws IOException + @return true if successful; + false if file does not exist or is a directory]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + + + core-default.xml]]> + + + + + + + + + + + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + + + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + + + + + + + + core-default.xml]]> + + + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + + + + core-default.xml]]> + + + + + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + + + + + + + core-default.xml]]> + + + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + + + + + + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + + + + + + core-default.xml]]> + + + + + + core-default.xml]]> + + + + + + + + core-default.xml]]> + + + + + + + + core-default.xml]]> + + + + + + + + core-default.xml]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + EnumSet.of(CreateFlag.CREATE, CreateFlag.APPEND) + +

      + + Use the CreateFlag as follows: +

        +
      1. CREATE - to create a file if it does not exist, + else throw FileAlreadyExists.
        2. +
      2. APPEND - to append to a file if it exists, + else throw FileNotFoundException.
        3. +
      3. OVERWRITE - to truncate a file if it exists, + else throw FileNotFoundException.
        4. +
      4. CREATE|APPEND - to create a file if it does not exist, + else append to an existing file.
        5. +
      5. CREATE|OVERWRITE - to create a file if it does not exist, + else overwrite an existing file.
        6. +
      6. SYNC_BLOCK - to force closed blocks to the disk device. + In addition {@link Syncable#hsync()} should be called after each write, + if true synchronous behavior is required.
        7. +
      7. LAZY_PERSIST - Create the block on transient storage (RAM) if + available.
        8. +
      8. APPEND_NEWBLOCK - Append data to a new block instead of end of the last + partial block.
        9. +
      + + Following combinations are not valid and will result in + {@link HadoopIllegalArgumentException}: +
        +
      1. APPEND|OVERWRITE
        2. +
      2. CREATE|APPEND|OVERWRITE
        3. +
      ]]> +       +       + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + absOrFqPath is not supported. + @throws IOException If the file system for absOrFqPath could + not be instantiated.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + defaultFsUri is not supported]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + NewWdir can be one of: +
        +
      • relative path: "foo/bar";
        • +
      • absolute without scheme: "/foo/bar"
        • +
      • fully qualified with scheme: "xx://auth/foo/bar"
        • +
      +
      + Illegal WDs: +
        +
      • relative with scheme: "xx:foo/bar"
        • +
      • non existent directory
        • +
      ]]> +       +       + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + f does not exist + @throws AccessControlException if access denied + @throws IOException If an IO Error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server + + RuntimeExceptions: + @throws InvalidPathException If path f is not valid]]> + + + + + + + + + + + + + + + + + + + + +
    4. Progress - to report progress on the operation - default null +
    5. Permission - umask is applied against permisssion: default is + FsPermissions:getDefault() + +
    6. CreateParent - create missing parent path; default is to not + to create parents +
    7. The defaults for the following are SS defaults of the file + server implementing the target path. Not all parameters make sense + for all kinds of file system - eg. localFS ignores Blocksize, + replication, checksum +
        +
      • BufferSize - buffersize used in FSDataOutputStream +
      • Blocksize - block size for file blocks +
      • ReplicationFactor - replication for blocks +
      • ChecksumParam - Checksum parameters. server default is used + if not specified. +
      +
+ + @return {@link FSDataOutputStream} for created file + + @throws AccessControlException If access is denied + @throws FileAlreadyExistsException If file f already exists + @throws FileNotFoundException If parent of f does not exist + and createParent is false + @throws ParentNotDirectoryException If parent of f is not a + directory. + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server + + RuntimeExceptions: + @throws InvalidPathException If path f is not valid]]> + + + + + + + + + + + + + + dir already + exists + @throws FileNotFoundException If parent of dir does not exist + and createParent is false + @throws ParentNotDirectoryException If parent of dir is not a + directory + @throws UnsupportedFileSystemException If file system for dir + is not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server + + RuntimeExceptions: + @throws InvalidPathException If path dir is not valid]]> + + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server + + RuntimeExceptions: + @throws InvalidPathException If path f is invalid]]> + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f + is not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + + +
  • Fails if path is a directory. +
  • Fails if path does not exist. +
  • Fails if path is not closed. +
  • Fails if new size is greater than current size. + + @param f The path to the file to be truncated + @param newLength The size the file is to be truncated to + + @return true if the file has been truncated to the desired + newLength and is immediately available to be reused for + write operations such as append, or + false if a background process of adjusting the length of + the last block has been started, and clients should wait for it to + complete before proceeding with further file updates. + + @throws AccessControlException If access is denied + @throws FileNotFoundException If file f does not exist + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + f does not exist + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + + + + + +
  • Fails if src is a file and dst is a directory. +
  • Fails if src is a directory and dst is a file. +
  • Fails if the parent of dst does not exist or is a file. + +

    + If OVERWRITE option is not passed as an argument, rename fails if the dst + already exists. +

    + If OVERWRITE option is passed as an argument, rename overwrites the dst if + it is a file or an empty directory. Rename fails if dst is a non-empty + directory. +

    + Note that atomicity of rename is dependent on the file system + implementation. Please refer to the file system documentation for details +

    + + @param src path to be renamed + @param dst new path after rename + + @throws AccessControlException If access is denied + @throws FileAlreadyExistsException If dst already exists and + options has {@link Options.Rename#OVERWRITE} + option false. + @throws FileNotFoundException If src does not exist + @throws ParentNotDirectoryException If parent of dst is not a + directory + @throws UnsupportedFileSystemException If file system for src + and dst is not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f + is not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server + + RuntimeExceptions: + @throws HadoopIllegalArgumentException If username or + groupname is invalid.]]> + + + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + f does not exist + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If an I/O error occurred]]> + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If the given path does not refer to a symlink + or an I/O error occurred]]> + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + + + + + + Given a path referring to a symlink of form: + + <---X---> + fs://host/A/B/link + <-----Y-----> + + In this path X is the scheme and authority that identify the file system, + and Y is the path leading up to the final path component "link". If Y is + a symlink itself then let Y' be the target of Y and X' be the scheme and + authority of Y'. Symlink targets may: + + 1. Fully qualified URIs + + fs://hostX/A/B/file Resolved according to the target file system. + + 2. Partially qualified URIs (eg scheme but no host) + + fs:///A/B/file Resolved according to the target file system. Eg resolving + a symlink to hdfs:///A results in an exception because + HDFS URIs must be fully qualified, while a symlink to + file:///A will not since Hadoop's local file systems + require partially qualified URIs. + + 3. Relative paths + + path Resolves to [Y'][path]. Eg if Y resolves to hdfs://host/A and path + is "../B/file" then [Y'][path] is hdfs://host/B/file + + 4. Absolute paths + + path Resolves to [X'][path]. Eg if Y resolves hdfs://host/A/B and path + is "/file" then [X][path] is hdfs://host/file + + + @param target the target of the symbolic link + @param link the path to be created that points to target + @param createParent if true then missing parent dirs are created if + false then parent must exist + + + @throws AccessControlException If access is denied + @throws FileAlreadyExistsException If file linkcode> already exists + @throws FileNotFoundException If target does not exist + @throws ParentNotDirectoryException If parent of link is not a + directory. + @throws UnsupportedFileSystemException If file system for + target or link is not supported + @throws IOException If an I/O error occurred]]> + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + + + + + + + + f does not exist + @throws UnsupportedFileSystemException If file system for f is + not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + f is + not supported + @throws IOException If an I/O error occurred + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + describing modifications + @throws IOException if an ACL could not be modified]]> + + + + + + + + describing entries to remove + @throws IOException if an ACL could not be modified]]> + + + + + + + + + + + + + + + + + + + + + + describing modifications, must include entries + for user, group, and others for compatibility with permission bits. + @throws IOException if an ACL could not be modified]]> + + + + + + + which returns each AclStatus + @throws IOException if an ACL could not be read]]> + + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to modify + @param name xattr name. + @param value xattr value. + @throws IOException]]> + + + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to modify + @param name xattr name. + @param value xattr value. + @param flag xattr set flag + @throws IOException]]> + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to get extended attribute + @param name xattr name. + @return byte[] xattr value. + @throws IOException]]> + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to get extended attributes + @return Map describing the XAttrs of the file or directory + @throws IOException]]> + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to get extended attributes + @param names XAttr names. + @return Map describing the XAttrs of the file or directory + @throws IOException]]> + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to remove extended attribute + @param name xattr name + @throws IOException]]> + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to get extended attributes + @return List of the XAttr names of the file or directory + @throws IOException]]> + + + + + + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + Exceptions applicable to file systems accessed over RPC: + @throws RpcClientException If an exception occurred in the RPC client + @throws RpcServerException If an exception occurred in the RPC server + @throws UnexpectedServerException If server implementation throws + undeclared exception to RPC server]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + *** Path Names *** +

    + + The Hadoop file system supports a URI name space and URI names. + It offers a forest of file systems that can be referenced using fully + qualified URIs. + Two common Hadoop file systems implementations are +

      +
    • the local file system: file:///path +
    • the hdfs file system hdfs://nnAddress:nnPort/path +
    + + While URI names are very flexible, it requires knowing the name or address + of the server. For convenience one often wants to access the default system + in one's environment without knowing its name/address. This has an + additional benefit that it allows one to change one's default fs + (e.g. admin moves application from cluster1 to cluster2). +

    + + To facilitate this, Hadoop supports a notion of a default file system. + The user can set his default file system, although this is + typically set up for you in your environment via your default config. + A default file system implies a default scheme and authority; slash-relative + names (such as /for/bar) are resolved relative to that default FS. + Similarly a user can also have working-directory-relative names (i.e. names + not starting with a slash). While the working directory is generally in the + same default FS, the wd can be in a different FS. +

    + Hence Hadoop path names can be one of: +

      +
    • fully qualified URI: scheme://authority/path +
    • slash relative names: /path relative to the default file system +
    • wd-relative names: path relative to the working dir +
    + Relative paths with scheme (scheme:foo/bar) are illegal. + +

    + ****The Role of the FileContext and configuration defaults**** +

    + The FileContext provides file namespace context for resolving file names; + it also contains the umask for permissions, In that sense it is like the + per-process file-related state in Unix system. + These two properties +

      +
    • default file system i.e your slash) +
    • umask +
    + in general, are obtained from the default configuration file + in your environment, (@see {@link Configuration}). + + No other configuration parameters are obtained from the default config as + far as the file context layer is concerned. All file system instances + (i.e. deployments of file systems) have default properties; we call these + server side (SS) defaults. Operation like create allow one to select many + properties: either pass them in as explicit parameters or use + the SS properties. +

    + The file system related SS defaults are +

      +
    • the home directory (default is "/user/userName") +
    • the initial wd (only for local fs) +
    • replication factor +
    • block size +
    • buffer size +
    • encryptDataTransfer +
    • checksum option. (checksumType and bytesPerChecksum) +
    + +

    + *** Usage Model for the FileContext class *** +

    + Example 1: use the default config read from the $HADOOP_CONFIG/core.xml. + Unspecified values come from core-defaults.xml in the release jar. +

      +
    • myFContext = FileContext.getFileContext(); // uses the default config + // which has your default FS +
    • myFContext.create(path, ...); +
    • myFContext.setWorkingDir(path) +
    • myFContext.open (path, ...); +
    + Example 2: Get a FileContext with a specific URI as the default FS +
      +
    • myFContext = FileContext.getFileContext(URI) +
    • myFContext.create(path, ...); + ... +
    + Example 3: FileContext with local file system as the default +
      +
    • myFContext = FileContext.getLocalFSFileContext() +
    • myFContext.create(path, ...); +
    • ... +
    + Example 4: Use a specific config, ignoring $HADOOP_CONFIG + Generally you should not need use a config unless you are doing +
      +
    • configX = someConfigSomeOnePassedToYou. +
    • myFContext = getFileContext(configX); // configX is not changed, + // is passed down +
    • myFContext.create(path, ...); +
    • ... +
    ]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This implementation throws an UnsupportedOperationException. + + @return the protocol scheme for the FileSystem.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + fs.scheme.class whose value names the FileSystem class. + The entire URI is passed to the FileSystem instance's initialize method.]]> + + + + + + + + + + + + + + + + + + fs.scheme.class whose value names the FileSystem class. + The entire URI is passed to the FileSystem instance's initialize method. + This always returns a new FileSystem object.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
  • Fails if src is a file and dst is a directory. +
  • Fails if src is a directory and dst is a file. +
  • Fails if the parent of dst does not exist or is a file. + +

    + If OVERWRITE option is not passed as an argument, rename fails + if the dst already exists. +

    + If OVERWRITE option is passed as an argument, rename overwrites + the dst if it is a file or an empty directory. Rename fails if dst is + a non-empty directory. +

    + Note that atomicity of rename is dependent on the file system + implementation. Please refer to the file system documentation for + details. This default implementation is non atomic. +

    + This method is deprecated since it is a temporary method added to + support the transition from FileSystem to FileContext for user + applications. + + @param src path to be renamed + @param dst new path after rename + @throws IOException on failure]]> + + + + + + + + +

  • Fails if path is a directory. +
  • Fails if path does not exist. +
  • Fails if path is not closed. +
  • Fails if new size is greater than current size. + + @param f The path to the file to be truncated + @param newLength The size the file is to be truncated to + + @return true if the file has been truncated to the desired + newLength and is immediately available to be reused for + write operations such as append, or + false if a background process of adjusting the length of + the last block has been started, and clients should wait for it to + complete before proceeding with further file updates.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Does not guarantee to return the List of files/directories status in a + sorted order. + @param f given path + @return the statuses of the files/directories in the given patch + @throws FileNotFoundException when the path does not exist; + IOException see specific implementation]]> + + + + + + + + + + + + + + + + + Does not guarantee to return the List of files/directories status in a + sorted order. + + @param f + a path name + @param filter + the user-supplied path filter + @return an array of FileStatus objects for the files under the given path + after applying the filter + @throws FileNotFoundException when the path does not exist; + IOException see specific implementation]]> + + + + + + + + + Does not guarantee to return the List of files/directories status in a + sorted order. + + @param files + a list of paths + @return a list of statuses for the files under the given paths after + applying the filter default Path filter + @throws FileNotFoundException when the path does not exist; + IOException see specific implementation]]> + + + + + + + + + + Does not guarantee to return the List of files/directories status in a + sorted order. + + @param files + a list of paths + @param filter + the user-supplied path filter + @return a list of statuses for the files under the given paths after + applying the filter + @throws FileNotFoundException when the path does not exist; + IOException see specific implementation]]> + + + + + + + Return all the files that match filePattern and are not checksum + files. Results are sorted by their names. + +

    + A filename pattern is composed of regular characters and + special pattern matching characters, which are: + +

    +
    +
    +

    +

    ? +
    Matches any single character. + +

    +

    * +
    Matches zero or more characters. + +

    +

    [abc] +
    Matches a single character from character set + {a,b,c}. + +

    +

    [a-b] +
    Matches a single character from the character range + {a...b}. Note that character a must be + lexicographically less than or equal to character b. + +

    +

    [^a] +
    Matches a single character that is not from character set or range + {a}. Note that the ^ character must occur + immediately to the right of the opening bracket. + +

    +

    \c +
    Removes (escapes) any special meaning of character c. + +

    +

    {ab,cd} +
    Matches a string from the string set {ab, cd} + +

    +

    {ab,c{de,fh}} +
    Matches a string from the string set {ab, cde, cfh} + +
    +
    +
    + + @param pathPattern a regular expression specifying a pth pattern + + @return an array of paths that match the path pattern + @throws IOException]]> +     +     + + + + + + + + + + + + + + f does not exist + @throws IOException If an I/O error occurred]]> + + + + + + + + + f does not exist + @throws IOException if any I/O error occurred]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + describing modifications + @throws IOException if an ACL could not be modified]]> + + + + + + + + describing entries to remove + @throws IOException if an ACL could not be modified]]> + + + + + + + + + + + + + + + + + + + + + + describing modifications, must include entries + for user, group, and others for compatibility with permission bits. + @throws IOException if an ACL could not be modified]]> + + + + + + + + + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to modify + @param name xattr name. + @param value xattr value. + @throws IOException]]> + + + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to modify + @param name xattr name. + @param value xattr value. + @param flag xattr set flag + @throws IOException]]> + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to get extended attribute + @param name xattr name. + @return byte[] xattr value. + @throws IOException]]> + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to get extended attributes + @return Map describing the XAttrs of the file or directory + @throws IOException]]> + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to get extended attributes + @param names XAttr names. + @return Map describing the XAttrs of the file or directory + @throws IOException]]> + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to get extended attributes + @return List of the XAttr names of the file or directory + @throws IOException]]> + + + + + + + + + Refer to the HDFS extended attributes user documentation for details. + + @param path Path to remove extended attribute + @param name xattr name + @throws IOException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This is a default method which is intended to be overridden by + subclasses. The default implementation returns an empty storage statistics + object.

    + + @return The StorageStatistics for this FileSystem instance. + Will never be null.]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + All user code that may potentially use the Hadoop Distributed + File System should be written to use a FileSystem object. The + Hadoop DFS is a multi-machine system that appears as a single + disk. It's useful because of its fault tolerance and potentially + very large capacity. + +

    + The local implementation is {@link LocalFileSystem} and distributed + implementation is DistributedFileSystem.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + caller's environment variables to use + for expansion + @return String[] with absolute path to new jar in position 0 and + unexpanded wild card entry path in position 1 + @throws IOException if there is an I/O error while writing the jar file]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + FilterFileSystem contains + some other file system, which it uses as + its basic file system, possibly transforming + the data along the way or providing additional + functionality. The class FilterFileSystem + itself simply overrides all methods of + FileSystem with versions that + pass all requests to the contained file + system. Subclasses of FilterFileSystem + may further override some of these methods + and may also provide additional methods + and fields.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + -1 + if there is no more data because the end of the stream has been + reached]]> + + + + + + + + + + length bytes have been read. + + @param position position in the input stream to seek + @param buffer buffer into which data is read + @param offset offset into the buffer in which data is written + @param length the number of bytes to read + @throws IOException IO problems + @throws EOFException If the end of stream is reached while reading. + If an exception is thrown an undetermined number + of bytes in the buffer may have been written.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + path is invalid]]> + + + + + + + + + + + + + + + + + + + + + + + @return file]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + and the scheme is null, and the authority + is null. + + @return whether the path is absolute and the URI has no scheme nor + authority parts]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if and only if pathname + should be included]]> + + + + + + + + + + + + + + Warning: Not all filesystems satisfy the thread-safety requirement. + @param position position within file + @param buffer destination buffer + @param offset offset in the buffer + @param length number of bytes to read + @return actual number of bytes read; -1 means "none" + @throws IOException IO problems.]]> + + + + + + + + + + Warning: Not all filesystems satisfy the thread-safety requirement. + @param position position within file + @param buffer destination buffer + @param offset offset in the buffer + @param length number of bytes to read + @throws IOException IO problems. + @throws EOFException the end of the data was reached before + the read operation completed]]> + + + + + + + + Warning: Not all filesystems satisfy the thread-safety requirement. + @param position position within file + @param buffer destination buffer + @throws IOException IO problems. + @throws EOFException the end of the data was reached before + the read operation completed]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <----15----> <----15----> <----15----> <-------18-------> + QUOTA REMAINING_QUATA SPACE_QUOTA SPACE_QUOTA_REM FILE_NAME]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + XAttr is byte[], this class is to + covert byte[] to some kind of string representation or convert back. + String representation is convenient for display and input. For example + display in screen as shell response and json response, input as http + or shell parameter.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @return ftp]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + A {@link FileSystem} backed by an FTP client provided by Apache Commons Net. +

    ]]> +     + + + + + + + + + + + + + + + + + + + (cause==null ? null : cause.toString()) (which + typically contains the class and detail message of cause). + @param cause the cause (which is saved for later retrieval by the + {@link #getCause()} method). (A null value is + permitted, and indicates that the cause is nonexistent or + unknown.)]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + But for removeAcl operation it will be false. i.e. AclSpec should + not contain permissions.
    + Example: "user:foo,group:bar" + @return Returns list of {@link AclEntry} parsed]]> +     +     + + + + + + The expected format of ACL entries in the string parameter is the same + format produced by the {@link #toStringStable()} method. + + @param aclStr + String representation of an ACL.
    + Example: "user:foo:rw-" + @param includePermission + for setAcl operations this will be true. i.e. Acl should include + permissions.
    + But for removeAcl operation it will be false. i.e. Acl should not + contain permissions.
    + Example: "user:foo,group:bar,mask::" + @return Returns an {@link AclEntry} object]]> +     +     + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + unmodifiable ordered list of all ACL entries]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + Recommended to use this API ONLY if client communicates with the old + NameNode, needs to pass the Permission for the path to get effective + permission, else use {@link AclStatus#getEffectivePermission(AclEntry)}. + @param entry AclEntry to get the effective action + @param permArg Permission for the path. However if the client is NOT + communicating with old namenode, then this argument will not have + any preference. + @return Returns the effective permission for the entry. + @throws IllegalArgumentException If the client communicating with old + namenode and permission is not passed as an argument.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + mode is invalid]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @return viewfs]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
  • /user -> hdfs://nnContainingUserDir/user +
  • /project/foo -> hdfs://nnProject1/projects/foo +
  • /project/bar -> hdfs://nnProject2/projects/bar +
  • /tmp -> hdfs://nnTmp/privateTmpForUserXXX + + + ViewFs is specified with the following URI: viewfs:/// +

    + To use viewfs one would typically set the default file system in the + config (i.e. fs.defaultFS < = viewfs:///) along with the + mount table config variables as described below. + +

    + ** Config variables to specify the mount table entries ** +

    + + The file system is initialized from the standard Hadoop config through + config variables. + See {@link FsConstants} for URI and Scheme constants; + See {@link Constants} for config var constants; + see {@link ConfigUtil} for convenient lib. + +

    + All the mount table config entries for view fs are prefixed by + fs.viewfs.mounttable. + For example the above example can be specified with the following + config variables: +

      +
    • fs.viewfs.mounttable.default.link./user= + hdfs://nnContainingUserDir/user +
    • fs.viewfs.mounttable.default.link./project/foo= + hdfs://nnProject1/projects/foo +
    • fs.viewfs.mounttable.default.link./project/bar= + hdfs://nnProject2/projects/bar +
    • fs.viewfs.mounttable.default.link./tmp= + hdfs://nnTmp/privateTmpForUserXXX +
    + + The default mount table (when no authority is specified) is + from config variables prefixed by fs.viewFs.mounttable.default + The authority component of a URI can be used to specify a different mount + table. For example, +
      +
    • viewfs://sanjayMountable/ +
    + is initialized from fs.viewFs.mounttable.sanjayMountable.* config variables. + +

    + **** Merge Mounts **** (NOTE: merge mounts are not implemented yet.) +

    + + One can also use "MergeMounts" to merge several directories (this is + sometimes called union-mounts or junction-mounts in the literature. + For example of the home directories are stored on say two file systems + (because they do not fit on one) then one could specify a mount + entry such as following merges two dirs: +

      +
    • /user -> hdfs://nnUser1/user,hdfs://nnUser2/user +
    + Such a mergeLink can be specified with the following config var where "," + is used as the separator for each of links to be merged: +
      +
    • fs.viewfs.mounttable.default.linkMerge./user= + hdfs://nnUser1/user,hdfs://nnUser1/user +
    + A special case of the merge mount is where mount table's root is merged + with the root (slash) of another file system: +
      +
    • fs.viewfs.mounttable.default.linkMergeSlash=hdfs://nn99/ +
    + In this cases the root of the mount table is merged with the root of + hdfs://nn99/ ]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Since these methods are often vendor- or device-specific, operators + may implement this interface in order to achieve fencing. +

    + Fencing is configured by the operator as an ordered list of methods to + attempt. Each method will be tried in turn, and the next in the list + will only be attempted if the previous one fails. See {@link NodeFencer} + for more information. +

    + If an implementation also implements {@link Configurable} then its + setConf method will be called upon instantiation.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + state (e.g ACTIVE/STANDBY) as well as + some additional information. + + @throws AccessControlException + if access is denied. + @throws IOException + if other errors happen + @see HAServiceStatus]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + hadoop.http.filter.initializers. + +

      +
    • StaticUserWebFilter - An authorization plugin that makes all +users a static configured user. +
    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + public class IntArrayWritable extends ArrayWritable { + public IntArrayWritable() { + super(IntWritable.class); + } + } + ]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + o is a ByteWritable with the same value.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + the class of the item + @param conf the configuration to store + @param item the object to be stored + @param keyName the name of the key to use + @throws IOException : forwards Exceptions from the underlying + {@link Serialization} classes.]]> + + + + + + + + + the class of the item + @param conf the configuration to use + @param keyName the name of the key to use + @param itemClass the class of the item + @return restored object + @throws IOException : forwards Exceptions from the underlying + {@link Serialization} classes.]]> + + + + + + + + + the class of the item + @param conf the configuration to use + @param items the objects to be stored + @param keyName the name of the key to use + @throws IndexOutOfBoundsException if the items array is empty + @throws IOException : forwards Exceptions from the underlying + {@link Serialization} classes.]]> + + + + + + + + + the class of the item + @param conf the configuration to use + @param keyName the name of the key to use + @param itemClass the class of the item + @return restored object + @throws IOException : forwards Exceptions from the underlying + {@link Serialization} classes.]]> + + + + + DefaultStringifier offers convenience methods to store/load objects to/from + the configuration. + + @param the class of the objects to stringify]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + o is a DoubleWritable with the same value.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + value argument is null or + its size is zero, the elementType argument must not be null. If + the argument value's size is bigger than zero, the argument + elementType is not be used. + + @param value + @param elementType]]> + + + + + value should not be null + or empty. + + @param value]]> + + + + + + + + + + + + + + value and elementType. If the value argument + is null or its size is zero, the elementType argument must not be + null. If the argument value's size is bigger than zero, the + argument elementType is not be used. + + @param value + @param elementType]]> + + + + + + + + + + + + + + + + + + + o is an EnumSetWritable with the same value, + or both are null.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + o is a FloatWritable with the same value.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + When two sequence files, which have same Key type but different Value + types, are mapped out to reduce, multiple Value types is not allowed. + In this case, this class can help you wrap instances with different types. +

    + +

    + Compared with ObjectWritable, this class is much more effective, + because ObjectWritable will append the class declaration as a String + into the output file in every Key-Value pair. +

    + +

    + Generic Writable implements {@link Configurable} interface, so that it will be + configured by the framework. The configuration is passed to the wrapped objects + implementing {@link Configurable} interface before deserialization. +

    + + how to use it:
    + 1. Write your own class, such as GenericObject, which extends GenericWritable.
    + 2. Implements the abstract method getTypes(), defines + the classes which will be wrapped in GenericObject in application. + Attention: this classes defined in getTypes() method, must + implement Writable interface. +

    + + The code looks like this: + 
    + public class GenericObject extends GenericWritable {
+ 
+   private static Class[] CLASSES = {
+               ClassType1.class, 
+               ClassType2.class,
+               ClassType3.class,
+               };
+
+   protected Class[] getTypes() {
+       return CLASSES;
+   }
+
+ }
+
    + + @since Nov 8, 2006]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + o is a IntWritable with the same value.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + closes the input and output streams + at the end. + + @param in InputStrem to read from + @param out OutputStream to write to + @param conf the Configuration object]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ignore any {@link IOException} or + null pointers. Must only be used for cleanup in exception handlers. + + @param log the log to record problems to at debug level. Can be null. + @param closeables the objects to close]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This is better than File#listDir because it does not ignore IOExceptions. + + @param dir The directory to list. + @param filter If non-null, the filter to use when listing + this directory. + @return The list of files in the directory. + + @throws IOException On I/O error]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + o is a LongWritable with the same value.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + A map is a directory containing two files, the data file, + containing all keys and values in the map, and a smaller index + file, containing a fraction of the keys. The fraction is determined by + {@link Writer#getIndexInterval()}. + +

    The index file is read entirely into memory. Thus key implementations + should try to keep themselves small. + +

    Map files are created by adding entries in-order. To maintain a large + database, perform updates by copying the previous version of a database and + merging in a sorted change list, to create a new version of the database in + a new file. Sorting large change lists can be done with {@link + SequenceFile.Sorter}.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + o is an MD5Hash whose digest contains the + same values.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + className by first finding + it in the specified conf. If the specified conf is null, + try load it directly.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + A {@link Comparator} that operates directly on byte representations of + objects. +

    + @param + @see DeserializerComparator]]> +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + SequenceFiles are flat files consisting of binary key/value + pairs. + +

    SequenceFile provides {@link SequenceFile.Writer}, + {@link SequenceFile.Reader} and {@link Sorter} classes for writing, + reading and sorting respectively.

    + + There are three SequenceFile Writers based on the + {@link CompressionType} used to compress key/value pairs: +
      +
    1. + Writer : Uncompressed records. +
      2. +
    2. + RecordCompressWriter : Record-compressed files, only compress + values. +
      3. +
    3. + BlockCompressWriter : Block-compressed files, both keys & + values are collected in 'blocks' + separately and compressed. The size of + the 'block' is configurable. +
    + +

    The actual compression algorithm used to compress key and/or values can be + specified by using the appropriate {@link CompressionCodec}.

    + +

    The recommended way is to use the static createWriter methods + provided by the SequenceFile to chose the preferred format.

    + +

    The {@link SequenceFile.Reader} acts as the bridge and can read any of the + above SequenceFile formats.

    + +

    SequenceFile Formats

    + +

    Essentially there are 3 different formats for SequenceFiles + depending on the CompressionType specified. All of them share a + common header described below. + +

    +
      +
    • + version - 3 bytes of magic header SEQ, followed by 1 byte of actual + version number (e.g. SEQ4 or SEQ6) +
      • +
    • + keyClassName -key class +
      • +
    • + valueClassName - value class +
      • +
    • + compression - A boolean which specifies if compression is turned on for + keys/values in this file. +
      • +
    • + blockCompression - A boolean which specifies if block-compression is + turned on for keys/values in this file. +
      • +
    • + compression codec - CompressionCodec class which is used for + compression of keys and/or values (if compression is + enabled). +
      • +
    • + metadata - {@link Metadata} for this file. +
      • +
    • + sync - A sync marker to denote end of the header. +
      • +
    + +
    Uncompressed SequenceFile Format
    +
      +
    • + Header +
      • +
    • + Record +
        +
      • Record length
        • +
      • Key length
        • +
      • Key
        • +
      • Value
        • +
      +
      • +
    • + A sync-marker every few 100 bytes or so. +
      • +
    + +
    Record-Compressed SequenceFile Format
    +
      +
    • + Header +
      • +
    • + Record +
        +
      • Record length
        • +
      • Key length
        • +
      • Key
        • +
      • Compressed Value
        • +
      +
      • +
    • + A sync-marker every few 100 bytes or so. +
      • +
    + +
    Block-Compressed SequenceFile Format
    +
      +
    • + Header +
      • +
    • + Record Block +
        +
      • Uncompressed number of records in the block
        • +
      • Compressed key-lengths block-size
        • +
      • Compressed key-lengths block
        • +
      • Compressed keys block-size
        • +
      • Compressed keys block
        • +
      • Compressed value-lengths block-size
        • +
      • Compressed value-lengths block
        • +
      • Compressed values block-size
        • +
      • Compressed values block
        • +
      +
      • +
    • + A sync-marker every block. +
      • +
    + +

    The compressed blocks of key lengths and value lengths consist of the + actual lengths of individual keys/values encoded in ZeroCompressedInteger + format.

    + + @see CompressionCodec]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + o is a ShortWritable with the same value.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + the class of the objects to stringify]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + position. Note that this + method avoids using the converter or doing String instantiation + @return the Unicode scalar value at position or -1 + if the position is invalid or points to a + trailing byte]]> + + + + + + + + + + what in the backing + buffer, starting as position start. The starting + position is measured in bytes and the return value is in + terms of byte position in the buffer. The backing buffer is + not converted to a string for this operation. + @return byte position of the first occurence of the search + string in the UTF-8 buffer or -1 if not found]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Note: For performance reasons, this call does not clear the + underlying byte array that is retrievable via {@link #getBytes()}. + In order to free the byte-array memory, call {@link #set(byte[])} + with an empty byte array (For example, new byte[0]).]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + o is a Text with the same contents.]]> + + + + + + + + + + + + + + + + + + + + + + + + + replace is true, then + malformed input is replaced with the + substitution character, which is U+FFFD. Otherwise the + method throws a MalformedInputException.]]> + + + + + + + + + + + + + + + replace is true, then + malformed input is replaced with the + substitution character, which is U+FFFD. Otherwise the + method throws a MalformedInputException. + @return ByteBuffer: bytes stores at ByteBuffer.array() + and length is ByteBuffer.limit()]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + In + addition, it provides methods for string traversal without converting the + byte array to a string.

    Also includes utilities for + serializing/deserialing a string, coding/decoding a string, checking if a + byte array contains valid UTF8 code, calculating the length of an encoded + string.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This is useful when a class may evolve, so that instances written by the + old version of the class may still be processed by the new version. To + handle this situation, {@link #readFields(DataInput)} + implementations should catch {@link VersionMismatchException}.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + o is a VIntWritable with the same value.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + o is a VLongWritable with the same value.]]> + + + + + + + + + + + + + + + + + + + + + + + + out. + + @param out DataOuput to serialize this object into. + @throws IOException]]> + + + + + + + in. + +

    For efficiency, implementations should attempt to re-use storage in the + existing object where possible.

    + + @param in DataInput to deseriablize this object from. + @throws IOException]]> +     + + + Any key or value type in the Hadoop Map-Reduce + framework implements this interface.

    + +

    Implementations typically implement a static read(DataInput) + method which constructs a new instance, calls {@link #readFields(DataInput)} + and returns the instance.

    + +

    Example:

    + 

    +     public class MyWritable implements Writable {
+       // Some data     
+       private int counter;
+       private long timestamp;
+       
+       public void write(DataOutput out) throws IOException {
+         out.writeInt(counter);
+         out.writeLong(timestamp);
+       }
+       
+       public void readFields(DataInput in) throws IOException {
+         counter = in.readInt();
+         timestamp = in.readLong();
+       }
+       
+       public static MyWritable read(DataInput in) throws IOException {
+         MyWritable w = new MyWritable();
+         w.readFields(in);
+         return w;
+       }
+     }
+

    ]]> +     + + + + + + + + WritableComparables can be compared to each other, typically + via Comparators. Any type which is to be used as a + key in the Hadoop Map-Reduce framework should implement this + interface.

    + +

    Note that hashCode() is frequently used in Hadoop to partition + keys. It's important that your implementation of hashCode() returns the same + result across different instances of the JVM. Note also that the default + hashCode() implementation in Object does not + satisfy this property.

    + +

    Example:

    + 

    +     public class MyWritableComparable implements WritableComparable {
+       // Some data
+       private int counter;
+       private long timestamp;
+       
+       public void write(DataOutput out) throws IOException {
+         out.writeInt(counter);
+         out.writeLong(timestamp);
+       }
+       
+       public void readFields(DataInput in) throws IOException {
+         counter = in.readInt();
+         timestamp = in.readLong();
+       }
+       
+       public int compareTo(MyWritableComparable o) {
+         int thisValue = this.value;
+         int thatValue = o.value;
+         return (thisValue < thatValue ? -1 : (thisValue==thatValue ? 0 : 1));
+       }
+
+       public int hashCode() {
+         final int prime = 31;
+         int result = 1;
+         result = prime * result + counter;
+         result = prime * result + (int) (timestamp ^ (timestamp >>> 32));
+         return result
+       }
+     }
+

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The default implementation reads the data into two {@link + WritableComparable}s (using {@link + Writable#readFields(DataInput)}, then calls {@link + #compare(WritableComparable,WritableComparable)}.]]> + + + + + + + The default implementation uses the natural ordering, calling {@link + Comparable#compareTo(Object)}.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This base implemenation uses the natural ordering. To define alternate + orderings, override {@link #compare(WritableComparable,WritableComparable)}. + +

    One may optimize compare-intensive operations by overriding + {@link #compare(byte[],int,int,byte[],int,int)}. Static utility methods are + provided to assist in optimized implementations of this method.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Enum type + @param in DataInput to read from + @param enumType Class type of Enum + @return Enum represented by String read from DataInput + @throws IOException]]> + + + + + + + + + + + + + + + + len number of bytes in input streamin + @param in input stream + @param len number of bytes to skip + @throws IOException when skipped less number of bytes]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + CompressionCodec for which to get the + Compressor + @param conf the Configuration object which contains confs for creating or reinit the compressor + @return Compressor for the given + CompressionCodec from the pool or a new one]]> + + + + + + + + + CompressionCodec for which to get the + Decompressor + @return Decompressor for the given + CompressionCodec the pool or a new one]]> + + + + + + Compressor to be returned to the pool]]> + + + + + + Decompressor to be returned to the + pool]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Codec aliases are case insensitive. +

    + The code alias is the short class name (without the package name). + If the short class name ends with 'Codec', then there are two aliases for + the codec, the complete short class name and the short class name without + the 'Codec' ending. For example for the 'GzipCodec' codec class name the + alias are 'gzip' and 'gzipcodec'. + + @param codecName the canonical class name of the codec + @return the codec object]]> + + + + + + + Codec aliases are case insensitive. +

    + The code alias is the short class name (without the package name). + If the short class name ends with 'Codec', then there are two aliases for + the codec, the complete short class name and the short class name without + the 'Codec' ending. For example for the 'GzipCodec' codec class name the + alias are 'gzip' and 'gzipcodec'. + + @param codecName the canonical class name of the codec + @return the codec class]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Implementations are assumed to be buffered. This permits clients to + reposition the underlying input stream then call {@link #resetState()}, + without having to also synchronize client buffers.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true indicating that more input data is required. + + @param b Input data + @param off Start offset + @param len Length]]> + + + + + true if the input data buffer is empty and + #setInput() should be called in order to provide more input.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if the end of the compressed + data output stream has been reached.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true indicating that more input data is required. + (Both native and non-native versions of various Decompressors require + that the data passed in via b[] remain unmodified until + the caller is explicitly notified--via {@link #needsInput()}--that the + buffer may be safely modified. With this requirement, an extra + buffer-copy can be avoided.) + + @param b Input data + @param off Start offset + @param len Length]]> + + + + + true if the input data buffer is empty and + {@link #setInput(byte[], int, int)} should be called to + provide more input. + + @return true if the input data buffer is empty and + {@link #setInput(byte[], int, int)} should be called in + order to provide more input.]]> + + + + + + + + + + + + + true if a preset dictionary is needed for decompression. + @return true if a preset dictionary is needed for decompression]]> + + + + + true if the end of the decompressed + data output stream has been reached. Indicates a concatenated data stream + when finished() returns true and {@link #getRemaining()} + returns a positive value. finished() will be reset with the + {@link #reset()} method. + @return true if the end of the decompressed + data output stream has been reached.]]> + + + + + + + + + + + + + + true and getRemaining() returns a positive value. If + {@link #finished()} returns true and getRemaining() returns + a zero value, indicates that the end of data stream has been reached and + is not a concatenated data stream. + @return The number of bytes remaining in the compressed data buffer.]]> + + + + + true and {@link #getRemaining()} returns a positive value, + reset() is called before processing of the next data stream in the + concatenated data stream. {@link #finished()} will be reset and will + return false when reset() is called.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

  • "none" - No compression. +
  • "lzo" - LZO compression. +
  • "gz" - GZIP compression. + ]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
  • Block Compression. +
  • Named meta data blocks. +
  • Sorted or unsorted keys. +
  • Seek by key or by file offset. + + The memory footprint of a TFile includes the following: +
      +
    • Some constant overhead of reading or writing a compressed block. +
        +
      • Each compressed block requires one compression/decompression codec for + I/O. +
      • Temporary space to buffer the key. +
      • Temporary space to buffer the value (for TFile.Writer only). Values are + chunk encoded, so that we buffer at most one chunk of user data. By default, + the chunk buffer is 1MB. Reading chunked value does not require additional + memory. +
      +
    • TFile index, which is proportional to the total number of Data Blocks. + The total amount of memory needed to hold the index can be estimated as + (56+AvgKeySize)*NumBlocks. +
    • MetaBlock index, which is proportional to the total number of Meta + Blocks.The total amount of memory needed to hold the index for Meta Blocks + can be estimated as (40+AvgMetaBlockName)*NumMetaBlock. +
    +

    + The behavior of TFile can be customized by the following variables through + Configuration: +

      +
    • tfile.io.chunk.size: Value chunk size. Integer (in bytes). Default + to 1MB. Values of the length less than the chunk size is guaranteed to have + known value length in read time (See + {@link TFile.Reader.Scanner.Entry#isValueLengthKnown()}). +
    • tfile.fs.output.buffer.size: Buffer size used for + FSDataOutputStream. Integer (in bytes). Default to 256KB. +
    • tfile.fs.input.buffer.size: Buffer size used for + FSDataInputStream. Integer (in bytes). Default to 256KB. +
    +

    + Suggestions on performance optimization. +

      +
    • Minimum block size. We recommend a setting of minimum block size between + 256KB to 1MB for general usage. Larger block size is preferred if files are + primarily for sequential access. However, it would lead to inefficient random + access (because there are more data to decompress). Smaller blocks are good + for random access, but require more memory to hold the block index, and may + be slower to create (because we must flush the compressor stream at the + conclusion of each data block, which leads to an FS I/O flush). Further, due + to the internal caching in Compression codec, the smallest possible block + size would be around 20KB-30KB. +
    • The current implementation does not offer true multi-threading for + reading. The implementation uses FSDataInputStream seek()+read(), which is + shown to be much faster than positioned-read call in single thread mode. + However, it also means that if multiple threads attempt to access the same + TFile (using multiple scanners) simultaneously, the actual I/O is carried out + sequentially even if they access different DFS blocks. +
    • Compression codec. Use "none" if the data is not very compressable (by + compressable, I mean a compression ratio at least 2:1). Generally, use "lzo" + as the starting point for experimenting. "gz" overs slightly better + compression ratio over "lzo" but requires 4x CPU to compress and 2x CPU to + decompress, comparing to "lzo". +
    • File system buffering, if the underlying FSDataInputStream and + FSDataOutputStream is already adequately buffered; or if applications + reads/writes keys and values in large buffers, we can reduce the sizes of + input/output buffering in TFile layer by setting the configuration parameters + "tfile.fs.input.buffer.size" and "tfile.fs.output.buffer.size". +
    + + Some design rationale behind TFile can be found at Hadoop-3315.]]> + + + + + + + + + + + Utils#writeVLong(out, n). + + @param out + output stream + @param n + The integer to be encoded + @throws IOException + @see Utils#writeVLong(DataOutput, long)]]> + + + + + + + + +
  • if n in [-32, 127): encode in one byte with the actual value. + Otherwise, +
  • if n in [-20*2^8, 20*2^8): encode in two bytes: byte[0] = n/256 - 52; + byte[1]=n&0xff. Otherwise, +
  • if n IN [-16*2^16, 16*2^16): encode in three bytes: byte[0]=n/2^16 - + 88; byte[1]=(n>>8)&0xff; byte[2]=n&0xff. Otherwise, +
  • if n in [-8*2^24, 8*2^24): encode in four bytes: byte[0]=n/2^24 - 112; + byte[1] = (n>>16)&0xff; byte[2] = (n>>8)&0xff; byte[3]=n&0xff. Otherwise: +
  • if n in [-2^31, 2^31): encode in five bytes: byte[0]=-125; byte[1] = + (n>>24)&0xff; byte[2]=(n>>16)&0xff; byte[3]=(n>>8)&0xff; byte[4]=n&0xff; +
  • if n in [-2^39, 2^39): encode in six bytes: byte[0]=-124; byte[1] = + (n>>32)&0xff; byte[2]=(n>>24)&0xff; byte[3]=(n>>16)&0xff; + byte[4]=(n>>8)&0xff; byte[5]=n&0xff +
  • if n in [-2^47, 2^47): encode in seven bytes: byte[0]=-123; byte[1] = + (n>>40)&0xff; byte[2]=(n>>32)&0xff; byte[3]=(n>>24)&0xff; + byte[4]=(n>>16)&0xff; byte[5]=(n>>8)&0xff; byte[6]=n&0xff; +
  • if n in [-2^55, 2^55): encode in eight bytes: byte[0]=-122; byte[1] = + (n>>48)&0xff; byte[2] = (n>>40)&0xff; byte[3]=(n>>32)&0xff; + byte[4]=(n>>24)&0xff; byte[5]=(n>>16)&0xff; byte[6]=(n>>8)&0xff; + byte[7]=n&0xff; +
  • if n in [-2^63, 2^63): encode in nine bytes: byte[0]=-121; byte[1] = + (n>>54)&0xff; byte[2] = (n>>48)&0xff; byte[3] = (n>>40)&0xff; + byte[4]=(n>>32)&0xff; byte[5]=(n>>24)&0xff; byte[6]=(n>>16)&0xff; + byte[7]=(n>>8)&0xff; byte[8]=n&0xff; + + + @param out + output stream + @param n + the integer number + @throws IOException]]> + + + + + + + (int)Utils#readVLong(in). + + @param in + input stream + @return the decoded integer + @throws IOException + + @see Utils#readVLong(DataInput)]]> + + + + + + + +
  • if (FB >= -32), return (long)FB; +
  • if (FB in [-72, -33]), return (FB+52)<<8 + NB[0]&0xff; +
  • if (FB in [-104, -73]), return (FB+88)<<16 + (NB[0]&0xff)<<8 + + NB[1]&0xff; +
  • if (FB in [-120, -105]), return (FB+112)<<24 + (NB[0]&0xff)<<16 + + (NB[1]&0xff)<<8 + NB[2]&0xff; +
  • if (FB in [-128, -121]), return interpret NB[FB+129] as a signed + big-endian integer. + + @param in + input stream + @return the decoded long integer. + @throws IOException]]> + + + + + + + + + + + + + + + + + + + + + + + + Type of the input key. + @param list + The list + @param key + The input key. + @param cmp + Comparator for the key. + @return The index to the desired element if it exists; or list.size() + otherwise.]]> + + + + + + + + + Type of the input key. + @param list + The list + @param key + The input key. + @param cmp + Comparator for the key. + @return The index to the desired element if it exists; or list.size() + otherwise.]]> + + + + + + + + Type of the input key. + @param list + The list + @param key + The input key. + @return The index to the desired element if it exists; or list.size() + otherwise.]]> + + + + + + + + Type of the input key. + @param list + The list + @param key + The input key. + @return The index to the desired element if it exists; or list.size() + otherwise.]]> + + + + + + + + + + + + + + + + + An experimental {@link Serialization} for Java {@link Serializable} classes. +

    + @see JavaSerializationComparator]]> +     +     + + + + + + + + + A {@link RawComparator} that uses a {@link JavaSerialization} + {@link Deserializer} to deserialize objects that are then compared via + their {@link Comparable} interfaces. +

    + @param + @see JavaSerialization]]> +     +     + + + + + + + + + + + + + +This package provides a mechanism for using different serialization frameworks +in Hadoop. The property "io.serializations" defines a list of +{@link org.apache.hadoop.io.serializer.Serialization}s that know how to create +{@link org.apache.hadoop.io.serializer.Serializer}s and +{@link org.apache.hadoop.io.serializer.Deserializer}s. +

    + +

    +To add a new serialization framework write an implementation of +{@link org.apache.hadoop.io.serializer.Serialization} and add its name to the +"io.serializations" property. +

    ]]> +     +     + + + + + + + + + + + + + + + + + + + avro.reflect.pkgs or implement + {@link AvroReflectSerializable} interface.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + +This package provides Avro serialization in Hadoop. This can be used to +serialize/deserialize Avro types in Hadoop. +

    + +

    +Use {@link org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization} for +serialization of classes generated by Avro's 'specific' compiler. +

    + +

    +Use {@link org.apache.hadoop.io.serializer.avro.AvroReflectSerialization} for +other classes. +{@link org.apache.hadoop.io.serializer.avro.AvroReflectSerialization} work for +any class which is either in the package list configured via +{@link org.apache.hadoop.io.serializer.avro.AvroReflectSerialization#AVRO_REFLECT_PACKAGES} +or implement {@link org.apache.hadoop.io.serializer.avro.AvroReflectSerializable} +interface. +

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + +The API is abstract so that it can be implemented on top of +a variety of metrics client libraries. The choice of +client library is a configuration option, and different +modules within the same application can use +different metrics implementation libraries. +

    +Sub-packages: +

    +
    org.apache.hadoop.metrics.spi
    +
    The abstract Server Provider Interface package. Those wishing to + integrate the metrics API with a particular metrics client library should + extend this package.
    + +
    org.apache.hadoop.metrics.file
    +
    An implementation package which writes the metric data to + a file, or sends it to the standard output stream.
    + +
    org.apache.hadoop.metrics.ganglia
    +
    An implementation package which sends metric data to + Ganglia.
    +
    + +

    Introduction to the Metrics API

    + +Here is a simple example of how to use this package to report a single +metric value: +
    +    private ContextFactory contextFactory = ContextFactory.getFactory();
+    
+    void reportMyMetric(float myMetric) {
+        MetricsContext myContext = contextFactory.getContext("myContext");
+        MetricsRecord myRecord = myContext.getRecord("myRecord");
+        myRecord.setMetric("myMetric", myMetric);
+        myRecord.update();
+    }
+
    + +In this example there are three names: +
    +
    myContext
    +
    The context name will typically identify either the application, or else a + module within an application or library.
    + +
    myRecord
    +
    The record name generally identifies some entity for which a set of + metrics are to be reported. For example, you could have a record named + "cacheStats" for reporting a number of statistics relating to the usage of + some cache in your application.
    + +
    myMetric
    +
    This identifies a particular metric. For example, you might have metrics + named "cache_hits" and "cache_misses". +
    +
    + +

    Tags

    + +In some cases it is useful to have multiple records with the same name. For +example, suppose that you want to report statistics about each disk on a computer. +In this case, the record name would be something like "diskStats", but you also +need to identify the disk which is done by adding a tag to the record. +The code could look something like this: +
    +    private MetricsRecord diskStats =
+            contextFactory.getContext("myContext").getRecord("diskStats");
+            
+    void reportDiskMetrics(String diskName, float diskBusy, float diskUsed) {
+        diskStats.setTag("diskName", diskName);
+        diskStats.setMetric("diskBusy", diskBusy);
+        diskStats.setMetric("diskUsed", diskUsed);
+        diskStats.update();
+    }
+
    + +

    Buffering and Callbacks

    + +Data is not sent immediately to the metrics system when +MetricsRecord.update() is called. Instead it is stored in an +internal table, and the contents of the table are sent periodically. +This can be important for two reasons: +
      +
    1. It means that a programmer is free to put calls to this API in an + inner loop, since updates can be very frequent without slowing down + the application significantly.
      2. +
    2. Some implementations can gain efficiency by combining many metrics + into a single UDP message.
      3. +
    + +The API provides a timer-based callback via the +registerUpdater() method. The benefit of this +versus using java.util.Timer is that the callbacks will be done +immediately before sending the data, making the data as current as possible. + +

    Configuration

    + +It is possible to programmatically examine and modify configuration data +before creating a context, like this: +
    +    ContextFactory factory = ContextFactory.getFactory();
+    ... examine and/or modify factory attributes ...
+    MetricsContext context = factory.getContext("myContext");
+
    +The factory attributes can be examined and modified using the following +ContextFactorymethods: +
      +
    • Object getAttribute(String attributeName)
      • +
    • String[] getAttributeNames()
      • +
    • void setAttribute(String name, Object value)
      • +
    • void removeAttribute(attributeName)
      • +
    + +

    +ContextFactory.getFactory() initializes the factory attributes by +reading the properties file hadoop-metrics.properties if it exists +on the class path. + +

    +A factory attribute named: +

    +contextName.class
+
    +should have as its value the fully qualified name of the class to be +instantiated by a call of the CodeFactory method +getContext(contextName). If this factory attribute is not +specified, the default is to instantiate +org.apache.hadoop.metrics.file.FileContext. + +

    +Other factory attributes are specific to a particular implementation of this +API and are documented elsewhere. For example, configuration attributes for +the file and Ganglia implementations can be found in the javadoc for +their respective packages.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Implementation of the metrics package that sends metric data to +Ganglia. +Programmers should not normally need to use this package directly. Instead +they should use org.hadoop.metrics. + +

    +These are the implementation specific factory attributes +(See ContextFactory.getFactory()): + +

    +
    contextName.servers
    +
    Space and/or comma separated sequence of servers to which UDP + messages should be sent.
    + +
    contextName.period
    +
    The period in seconds on which the metric data is sent to the + server(s).
    + +
    contextName.multicast
    +
    Enable multicast for Ganglia
    + +
    contextName.multicast.ttl
    +
    TTL for multicast packets
    + +
    contextName.units.recordName.metricName
    +
    The units for the specified metric in the specified record.
    + +
    contextName.slope.recordName.metricName
    +
    The slope for the specified metric in the specified record.
    + +
    contextName.tmax.recordName.metricName
    +
    The tmax for the specified metric in the specified record.
    + +
    contextName.dmax.recordName.metricName
    +
    The dmax for the specified metric in the specified record.
    + +
    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + contextName.tableName. The returned map consists of + those attributes with the contextName and tableName stripped off.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + recordName. + Throws an exception if the metrics implementation is configured with a fixed + set of record names and recordName is not in that set. + + @param recordName the name of the record + @throws MetricsException if recordName conflicts with configuration data]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This class implements the internal table of metric data, and the timer + on which data is to be sent to the metrics system. Subclasses must + override the abstract emitRecord method in order to transmit + the data.

    + + @deprecated Use org.apache.hadoop.metrics2 package instead.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + update + and remove(). + + @deprecated Use {@link org.apache.hadoop.metrics2.impl.MetricsRecordImpl} + instead.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + hostname or hostname:port. If + the specs string is null, defaults to localhost:defaultPort. + + @return a list of InetSocketAddress objects.]]> + + + + + + + + + org.apache.hadoop.metrics.file and +org.apache.hadoop.metrics.ganglia.

    + +Plugging in an implementation involves writing a concrete subclass of +AbstractMetricsContext. The subclass should get its + configuration information using the getAttribute(attributeName) + method.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Implementations of this interface consume the {@link MetricsRecord} generated + from {@link MetricsSource}. It registers with {@link MetricsSystem} which + periodically pushes the {@link MetricsRecord} to the sink using + {@link #putMetrics(MetricsRecord)} method. If the implementing class also + implements {@link Closeable}, then the MetricsSystem will close the sink when + it is stopped.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + the actual type of the source object + @param source object to register + @return the source object + @exception MetricsException]]> + + + + + + + + the actual type of the source object + @param source object to register + @param name of the source. Must be unique or null (then extracted from + the annotations of the source object.) + @param desc the description of the source (or null. See above.) + @return the source object + @exception MetricsException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + CollectD StatsD plugin). +
    + To configure this plugin, you will need to add the following + entries to your hadoop-metrics2.properties file: +
    + 

    + *.sink.statsd.class=org.apache.hadoop.metrics2.sink.StatsDSink
+ [prefix].sink.statsd.server.host=
+ [prefix].sink.statsd.server.port=
+ [prefix].sink.statsd.skip.hostname=true|false (optional)
+ [prefix].sink.statsd.service.name=NameNode (name you want for service)
+
    ]]> +     +     + +     + + + + + + + + + + + + + + + ,name=" + Where the and are the supplied parameters + + @param serviceName + @param nameName + @param theMbean - the MBean to register + @return the named used to register the MBean]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + hostname or hostname:port. If + the specs string is null, defaults to localhost:defaultPort. + + @param specs server specs (see description) + @param defaultPort the default port if not specified + @return a list of InetSocketAddress objects.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This method is used when parts of Hadoop need know whether to apply + single rack vs multi-rack policies, such as during block placement. + Such algorithms behave differently if they are on multi-switch systems. +

    + + @return true if the mapping thinks that it is on a single switch]]> +     +     + + + + + + + + + + + + + + + + + This predicate simply assumes that all mappings not derived from + this class are multi-switch. + @param mapping the mapping to query + @return true if the base class says it is single switch, or the mapping + is not derived from this class.]]> + + + + It is not mandatory to + derive {@link DNSToSwitchMapping} implementations from it, but it is strongly + recommended, as it makes it easy for the Hadoop developers to add new methods + to this base class that are automatically picked up by all implementations. +

    + + This class does not extend the Configured + base class, and should not be changed to do so, as it causes problems + for subclasses. The constructor of the Configured calls + the {@link #setConf(Configuration)} method, which will call into the + subclasses before they have been fully constructed.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + If a name cannot be resolved to a rack, the implementation + should return {@link NetworkTopology#DEFAULT_RACK}. This + is what the bundled implementations do, though it is not a formal requirement + + @param names the list of hosts to resolve (can be empty) + @return list of resolved network paths. + If names is empty, the returned list is also empty]]> + + + + + + + + + + + + + + + + + + + + + + + + Calling {@link #setConf(Configuration)} will trigger a + re-evaluation of the configuration settings and so be used to + set up the mapping script.]]> + + + + + + + + + + + + + + + + + + + + + This will get called in the superclass constructor, so a check is needed + to ensure that the raw mapping is defined before trying to relaying a null + configuration. + @param conf]]> + + + + + + + + + + It contains a static class RawScriptBasedMapping that performs + the work: reading the configuration parameters, executing any defined + script, handling errors and such like. The outer + class extends {@link CachedDNSToSwitchMapping} to cache the delegated + queries. +

    + This DNS mapper's {@link #isSingleSwitch()} predicate returns + true if and only if a script is defined.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Simple {@link DNSToSwitchMapping} implementation that reads a 2 column text + file. The columns are separated by whitespace. The first column is a DNS or + IP address and the second column specifies the rack where the address maps. +

    +

    + This class uses the configuration parameter {@code + net.topology.table.file.name} to locate the mapping file. +

    +

    + Calls to {@link #resolve(List)} will look up the address as defined in the + mapping file. If no entry corresponding to the address is found, the value + {@code /default-rack} is returned. +

    ]]> +     +     + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Avro.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Avro.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + =} getCount(). + @param newCapacity The new capacity in bytes.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Avro.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Avro.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Avro.]]> + + + + + + + + + + + + Index idx = startVector(...); + while (!idx.done()) { + .... // read element of a vector + idx.incr(); + } + + + @deprecated Replaced by Avro.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Avro.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + Avro.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Avro.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Avro.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Avro.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Avro.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Avro.]]> + + + + + + (DEPRECATED) Hadoop record I/O contains classes and a record description language + translator for simplifying serialization and deserialization of records in a + language-neutral manner. +

    + +

    + DEPRECATED: Replaced by Avro. +

    + +

    Introduction

    + + Software systems of any significant complexity require mechanisms for data +interchange with the outside world. These interchanges typically involve the +marshaling and unmarshaling of logical units of data to and from data streams +(files, network connections, memory buffers etc.). Applications usually have +some code for serializing and deserializing the data types that they manipulate +embedded in them. The work of serialization has several features that make +automatic code generation for it worthwhile. Given a particular output encoding +(binary, XML, etc.), serialization of primitive types and simple compositions +of primitives (structs, vectors etc.) is a very mechanical task. Manually +written serialization code can be susceptible to bugs especially when records +have a large number of fields or a record definition changes between software +versions. Lastly, it can be very useful for applications written in different +programming languages to be able to share and interchange data. This can be +made a lot easier by describing the data records manipulated by these +applications in a language agnostic manner and using the descriptions to derive +implementations of serialization in multiple target languages. + +This document describes Hadoop Record I/O, a mechanism that is aimed +at +
      +
    • enabling the specification of simple serializable data types (records) +
    • enabling the generation of code in multiple target languages for +marshaling and unmarshaling such types +
    • providing target language specific support that will enable application +programmers to incorporate generated code into their applications +
    + +The goals of Hadoop Record I/O are similar to those of mechanisms such as XDR, +ASN.1, PADS and ICE. While these systems all include a DDL that enables +the specification of most record types, they differ widely in what else they +focus on. The focus in Hadoop Record I/O is on data marshaling and +multi-lingual support. We take a translator-based approach to serialization. +Hadoop users have to describe their data in a simple data description +language. The Hadoop DDL translator rcc generates code that users +can invoke in order to read/write their data from/to simple stream +abstractions. Next we list explicitly some of the goals and non-goals of +Hadoop Record I/O. + + +

    Goals

    + +
      +
    • Support for commonly used primitive types. Hadoop should include as +primitives commonly used builtin types from programming languages we intend to +support. + +
    • Support for common data compositions (including recursive compositions). +Hadoop should support widely used composite types such as structs and +vectors. + +
    • Code generation in multiple target languages. Hadoop should be capable of +generating serialization code in multiple target languages and should be +easily extensible to new target languages. The initial target languages are +C++ and Java. + +
    • Support for generated target languages. Hadooop should include support +in the form of headers, libraries, packages for supported target languages +that enable easy inclusion and use of generated code in applications. + +
    • Support for multiple output encodings. Candidates include +packed binary, comma-separated text, XML etc. + +
    • Support for specifying record types in a backwards/forwards compatible +manner. This will probably be in the form of support for optional fields in +records. This version of the document does not include a description of the +planned mechanism, we intend to include it in the next iteration. + +
    + +

    Non-Goals

    + +
      +
    • Serializing existing arbitrary C++ classes. +
    • Serializing complex data structures such as trees, linked lists etc. +
    • Built-in indexing schemes, compression, or check-sums. +
    • Dynamic construction of objects from an XML schema. +
    + +The remainder of this document describes the features of Hadoop record I/O +in more detail. Section 2 describes the data types supported by the system. +Section 3 lays out the DDL syntax with some examples of simple records. +Section 4 describes the process of code generation with rcc. Section 5 +describes target language mappings and support for Hadoop types. We include a +fairly complete description of C++ mappings with intent to include Java and +others in upcoming iterations of this document. The last section talks about +supported output encodings. + + +

    Data Types and Streams

    + +This section describes the primitive and composite types supported by Hadoop. +We aim to support a set of types that can be used to simply and efficiently +express a wide range of record types in different programming languages. + +

    Primitive Types

    + +For the most part, the primitive types of Hadoop map directly to primitive +types in high level programming languages. Special cases are the +ustring (a Unicode string) and buffer types, which we believe +find wide use and which are usually implemented in library code and not +available as language built-ins. Hadoop also supplies these via library code +when a target language built-in is not present and there is no widely +adopted "standard" implementation. The complete list of primitive types is: + +
      +
    • byte: An 8-bit unsigned integer. +
    • boolean: A boolean value. +
    • int: A 32-bit signed integer. +
    • long: A 64-bit signed integer. +
    • float: A single precision floating point number as described by + IEEE-754. +
    • double: A double precision floating point number as described by + IEEE-754. +
    • ustring: A string consisting of Unicode characters. +
    • buffer: An arbitrary sequence of bytes. +
    + + +

    Composite Types

    +Hadoop supports a small set of composite types that enable the description +of simple aggregate types and containers. A composite type is serialized +by sequentially serializing it constituent elements. The supported +composite types are: + +
      + +
    • record: An aggregate type like a C-struct. This is a list of +typed fields that are together considered a single unit of data. A record +is serialized by sequentially serializing its constituent fields. In addition +to serialization a record has comparison operations (equality and less-than) +implemented for it, these are defined as memberwise comparisons. + +
    • vector: A sequence of entries of the same data type, primitive +or composite. + +
    • map: An associative container mapping instances of a key type to +instances of a value type. The key and value types may themselves be primitive +or composite types. + +
    + +

    Streams

    + +Hadoop generates code for serializing and deserializing record types to +abstract streams. For each target language Hadoop defines very simple input +and output stream interfaces. Application writers can usually develop +concrete implementations of these by putting a one method wrapper around +an existing stream implementation. + + +

    DDL Syntax and Examples

    + +We now describe the syntax of the Hadoop data description language. This is +followed by a few examples of DDL usage. + +

    Hadoop DDL Syntax

    + +
    
+recfile = *include module *record
+include = "include" path
+path = (relative-path / absolute-path)
+module = "module" module-name
+module-name = name *("." name)
+record := "class" name "{" 1*(field) "}"
+field := type name ";"
+name :=  ALPHA (ALPHA / DIGIT / "_" )*
+type := (ptype / ctype)
+ptype := ("byte" / "boolean" / "int" |
+          "long" / "float" / "double"
+          "ustring" / "buffer")
+ctype := (("vector" "<" type ">") /
+          ("map" "<" type "," type ">" ) ) / name)
+
    + +A DDL file describes one or more record types. It begins with zero or +more include declarations, a single mandatory module declaration +followed by zero or more class declarations. The semantics of each of +these declarations are described below: + +
      + +
    • include: An include declaration specifies a DDL file to be +referenced when generating code for types in the current DDL file. Record types +in the current compilation unit may refer to types in all included files. +File inclusion is recursive. An include does not trigger code +generation for the referenced file. + +
    • module: Every Hadoop DDL file must have a single module +declaration that follows the list of includes and precedes all record +declarations. A module declaration identifies a scope within which +the names of all types in the current file are visible. Module names are +mapped to C++ namespaces, Java packages etc. in generated code. + +
    • class: Records types are specified through class +declarations. A class declaration is like a Java class declaration. +It specifies a named record type and a list of fields that constitute records +of the type. Usage is illustrated in the following examples. + +
    + +

    Examples

    + +
      +
    • A simple DDL file links.jr with just one record declaration. +
      
+module links {
+    class Link {
+        ustring URL;
+        boolean isRelative;
+        ustring anchorText;
+    };
+}
+
      + +
    • A DDL file outlinks.jr which includes another +
      
+include "links.jr"
+
+module outlinks {
+    class OutLinks {
+        ustring baseURL;
+        vector outLinks;
+    };
+}
+
      +
    + +

    Code Generation

    + +The Hadoop translator is written in Java. Invocation is done by executing a +wrapper shell script named named rcc. It takes a list of +record description files as a mandatory argument and an +optional language argument (the default is Java) --language or +-l. Thus a typical invocation would look like: +
    
+$ rcc -l C++  ...
+
    + + +

    Target Language Mappings and Support

    + +For all target languages, the unit of code generation is a record type. +For each record type, Hadoop generates code for serialization and +deserialization, record comparison and access to record members. + +

    C++

    + +Support for including Hadoop generated C++ code in applications comes in the +form of a header file recordio.hh which needs to be included in source +that uses Hadoop types and a library librecordio.a which applications need +to be linked with. The header declares the Hadoop C++ namespace which defines +appropriate types for the various primitives, the basic interfaces for +records and streams and enumerates the supported serialization encodings. +Declarations of these interfaces and a description of their semantics follow: + +
    
+namespace hadoop {
+
+  enum RecFormat { kBinary, kXML, kCSV };
+
+  class InStream {
+  public:
+    virtual ssize_t read(void *buf, size_t n) = 0;
+  };
+
+  class OutStream {
+  public:
+    virtual ssize_t write(const void *buf, size_t n) = 0;
+  };
+
+  class IOError : public runtime_error {
+  public:
+    explicit IOError(const std::string& msg);
+  };
+
+  class IArchive;
+  class OArchive;
+
+  class RecordReader {
+  public:
+    RecordReader(InStream& in, RecFormat fmt);
+    virtual ~RecordReader(void);
+
+    virtual void read(Record& rec);
+  };
+
+  class RecordWriter {
+  public:
+    RecordWriter(OutStream& out, RecFormat fmt);
+    virtual ~RecordWriter(void);
+
+    virtual void write(Record& rec);
+  };
+
+
+  class Record {
+  public:
+    virtual std::string type(void) const = 0;
+    virtual std::string signature(void) const = 0;
+  protected:
+    virtual bool validate(void) const = 0;
+
+    virtual void
+    serialize(OArchive& oa, const std::string& tag) const = 0;
+
+    virtual void
+    deserialize(IArchive& ia, const std::string& tag) = 0;
+  };
+}
+
    + +
      + +
    • RecFormat: An enumeration of the serialization encodings supported +by this implementation of Hadoop. + +
    • InStream: A simple abstraction for an input stream. This has a +single public read method that reads n bytes from the stream into +the buffer buf. Has the same semantics as a blocking read system +call. Returns the number of bytes read or -1 if an error occurs. + +
    • OutStream: A simple abstraction for an output stream. This has a +single write method that writes n bytes to the stream from the +buffer buf. Has the same semantics as a blocking write system +call. Returns the number of bytes written or -1 if an error occurs. + +
    • RecordReader: A RecordReader reads records one at a time from +an underlying stream in a specified record format. The reader is instantiated +with a stream and a serialization format. It has a read method that +takes an instance of a record and deserializes the record from the stream. + +
    • RecordWriter: A RecordWriter writes records one at a +time to an underlying stream in a specified record format. The writer is +instantiated with a stream and a serialization format. It has a +write method that takes an instance of a record and serializes the +record to the stream. + +
    • Record: The base class for all generated record types. This has two +public methods type and signature that return the typename and the +type signature of the record. + +
    + +Two files are generated for each record file (note: not for each record). If a +record file is named "name.jr", the generated files are +"name.jr.cc" and "name.jr.hh" containing serialization +implementations and record type declarations respectively. + +For each record in the DDL file, the generated header file will contain a +class definition corresponding to the record type, method definitions for the +generated type will be present in the '.cc' file. The generated class will +inherit from the abstract class hadoop::Record. The DDL files +module declaration determines the namespace the record belongs to. +Each '.' delimited token in the module declaration results in the +creation of a namespace. For instance, the declaration module docs.links +results in the creation of a docs namespace and a nested +docs::links namespace. In the preceding examples, the Link class +is placed in the links namespace. The header file corresponding to +the links.jr file will contain: + +
    
+namespace links {
+  class Link : public hadoop::Record {
+    // ....
+  };
+};
+
    + +Each field within the record will cause the generation of a private member +declaration of the appropriate type in the class declaration, and one or more +acccessor methods. The generated class will implement the serialize and +deserialize methods defined in hadoop::Record+. It will also +implement the inspection methods type and signature from +hadoop::Record. A default constructor and virtual destructor will also +be generated. Serialization code will read/write records into streams that +implement the hadoop::InStream and the hadoop::OutStream interfaces. + +For each member of a record an accessor method is generated that returns +either the member or a reference to the member. For members that are returned +by value, a setter method is also generated. This is true for primitive +data members of the types byte, int, long, boolean, float and +double. For example, for a int field called MyField the folowing +code is generated. + +
    
+...
+private:
+  int32_t mMyField;
+  ...
+public:
+  int32_t getMyField(void) const {
+    return mMyField;
+  };
+
+  void setMyField(int32_t m) {
+    mMyField = m;
+  };
+  ...
+
    + +For a ustring or buffer or composite field. The generated code +only contains accessors that return a reference to the field. A const +and a non-const accessor are generated. For example: + +
    
+...
+private:
+  std::string mMyBuf;
+  ...
+public:
+
+  std::string& getMyBuf() {
+    return mMyBuf;
+  };
+
+  const std::string& getMyBuf() const {
+    return mMyBuf;
+  };
+  ...
+
    + +

    Examples

    + +Suppose the inclrec.jr file contains: +
    
+module inclrec {
+    class RI {
+        int      I32;
+        double   D;
+        ustring  S;
+    };
+}
+
    + +and the testrec.jr file contains: + +
    
+include "inclrec.jr"
+module testrec {
+    class R {
+        vector VF;
+        RI            Rec;
+        buffer        Buf;
+    };
+}
+
    + +Then the invocation of rcc such as: +
    
+$ rcc -l c++ inclrec.jr testrec.jr
+
    +will result in generation of four files: +inclrec.jr.{cc,hh} and testrec.jr.{cc,hh}. + +The inclrec.jr.hh will contain: + +
    
+#ifndef _INCLREC_JR_HH_
+#define _INCLREC_JR_HH_
+
+#include "recordio.hh"
+
+namespace inclrec {
+  
+  class RI : public hadoop::Record {
+
+  private:
+
+    int32_t      I32;
+    double       D;
+    std::string  S;
+
+  public:
+
+    RI(void);
+    virtual ~RI(void);
+
+    virtual bool operator==(const RI& peer) const;
+    virtual bool operator<(const RI& peer) const;
+
+    virtual int32_t getI32(void) const { return I32; }
+    virtual void setI32(int32_t v) { I32 = v; }
+
+    virtual double getD(void) const { return D; }
+    virtual void setD(double v) { D = v; }
+
+    virtual std::string& getS(void) const { return S; }
+    virtual const std::string& getS(void) const { return S; }
+
+    virtual std::string type(void) const;
+    virtual std::string signature(void) const;
+
+  protected:
+
+    virtual void serialize(hadoop::OArchive& a) const;
+    virtual void deserialize(hadoop::IArchive& a);
+  };
+} // end namespace inclrec
+
+#endif /* _INCLREC_JR_HH_ */
+
+
    + +The testrec.jr.hh file will contain: + + +
    
+
+#ifndef _TESTREC_JR_HH_
+#define _TESTREC_JR_HH_
+
+#include "inclrec.jr.hh"
+
+namespace testrec {
+  class R : public hadoop::Record {
+
+  private:
+
+    std::vector VF;
+    inclrec::RI        Rec;
+    std::string        Buf;
+
+  public:
+
+    R(void);
+    virtual ~R(void);
+
+    virtual bool operator==(const R& peer) const;
+    virtual bool operator<(const R& peer) const;
+
+    virtual std::vector& getVF(void) const;
+    virtual const std::vector& getVF(void) const;
+
+    virtual std::string& getBuf(void) const ;
+    virtual const std::string& getBuf(void) const;
+
+    virtual inclrec::RI& getRec(void) const;
+    virtual const inclrec::RI& getRec(void) const;
+    
+    virtual bool serialize(hadoop::OutArchive& a) const;
+    virtual bool deserialize(hadoop::InArchive& a);
+    
+    virtual std::string type(void) const;
+    virtual std::string signature(void) const;
+  };
+}; // end namespace testrec
+#endif /* _TESTREC_JR_HH_ */
+
+
    + +

    Java

    + +Code generation for Java is similar to that for C++. A Java class is generated +for each record type with private members corresponding to the fields. Getters +and setters for fields are also generated. Some differences arise in the +way comparison is expressed and in the mapping of modules to packages and +classes to files. For equality testing, an equals method is generated +for each record type. As per Java requirements a hashCode method is also +generated. For comparison a compareTo method is generated for each +record type. This has the semantics as defined by the Java Comparable +interface, that is, the method returns a negative integer, zero, or a positive +integer as the invoked object is less than, equal to, or greater than the +comparison parameter. + +A .java file is generated per record type as opposed to per DDL +file as in C++. The module declaration translates to a Java +package declaration. The module name maps to an identical Java package +name. In addition to this mapping, the DDL compiler creates the appropriate +directory hierarchy for the package and places the generated .java +files in the correct directories. + +

    Mapping Summary

    + +
    
+DDL Type        C++ Type            Java Type 
+
+boolean         bool                boolean
+byte            int8_t              byte
+int             int32_t             int
+long            int64_t             long
+float           float               float
+double          double              double
+ustring         std::string         java.lang.String
+buffer          std::string         org.apache.hadoop.record.Buffer
+class type      class type          class type
+vector    std::vector   java.util.ArrayList
+map  std::map java.util.TreeMap
+
    + +

    Data encodings

    + +This section describes the format of the data encodings supported by Hadoop. +Currently, three data encodings are supported, namely binary, CSV and XML. + +

    Binary Serialization Format

    + +The binary data encoding format is fairly dense. Serialization of composite +types is simply defined as a concatenation of serializations of the constituent +elements (lengths are included in vectors and maps). + +Composite types are serialized as follows: +
      +
    • class: Sequence of serialized members. +
    • vector: The number of elements serialized as an int. Followed by a +sequence of serialized elements. +
    • map: The number of key value pairs serialized as an int. Followed +by a sequence of serialized (key,value) pairs. +
    + +Serialization of primitives is more interesting, with a zero compression +optimization for integral types and normalization to UTF-8 for strings. +Primitive types are serialized as follows: + +
      +
    • byte: Represented by 1 byte, as is. +
    • boolean: Represented by 1-byte (0 or 1) +
    • int/long: Integers and longs are serialized zero compressed. +Represented as 1-byte if -120 <= value < 128. Otherwise, serialized as a +sequence of 2-5 bytes for ints, 2-9 bytes for longs. The first byte represents +the number of trailing bytes, N, as the negative number (-120-N). For example, +the number 1024 (0x400) is represented by the byte sequence 'x86 x04 x00'. +This doesn't help much for 4-byte integers but does a reasonably good job with +longs without bit twiddling. +
    • float/double: Serialized in IEEE 754 single and double precision +format in network byte order. This is the format used by Java. +
    • ustring: Serialized as 4-byte zero compressed length followed by +data encoded as UTF-8. Strings are normalized to UTF-8 regardless of native +language representation. +
    • buffer: Serialized as a 4-byte zero compressed length followed by the +raw bytes in the buffer. +
    + + +

    CSV Serialization Format

    + +The CSV serialization format has a lot more structure than the "standard" +Excel CSV format, but we believe the additional structure is useful because + +
      +
    • it makes parsing a lot easier without detracting too much from legibility +
    • the delimiters around composites make it obvious when one is reading a +sequence of Hadoop records +
    + +Serialization formats for the various types are detailed in the grammar that +follows. The notable feature of the formats is the use of delimiters for +indicating the certain field types. + +
      +
    • A string field begins with a single quote ('). +
    • A buffer field begins with a sharp (#). +
    • A class, vector or map begins with 's{', 'v{' or 'm{' respectively and +ends with '}'. +
    + +The CSV format can be described by the following grammar: + +
    
+record = primitive / struct / vector / map
+primitive = boolean / int / long / float / double / ustring / buffer
+
+boolean = "T" / "F"
+int = ["-"] 1*DIGIT
+long = ";" ["-"] 1*DIGIT
+float = ["-"] 1*DIGIT "." 1*DIGIT ["E" / "e" ["-"] 1*DIGIT]
+double = ";" ["-"] 1*DIGIT "." 1*DIGIT ["E" / "e" ["-"] 1*DIGIT]
+
+ustring = "'" *(UTF8 char except NULL, LF, % and , / "%00" / "%0a" / "%25" / "%2c" )
+
+buffer = "#" *(BYTE except NULL, LF, % and , / "%00" / "%0a" / "%25" / "%2c" )
+
+struct = "s{" record *("," record) "}"
+vector = "v{" [record *("," record)] "}"
+map = "m{" [*(record "," record)] "}"
+
    + +

    XML Serialization Format

    + +The XML serialization format is the same used by Apache XML-RPC +(http://ws.apache.org/xmlrpc/types.html). This is an extension of the original +XML-RPC format and adds some additional data types. All record I/O types are +not directly expressible in this format, and access to a DDL is required in +order to convert these to valid types. All types primitive or composite are +represented by <value> elements. The particular XML-RPC type is +indicated by a nested element in the <value> element. The encoding for +records is always UTF-8. Primitive types are serialized as follows: + +
      +
    • byte: XML tag <ex:i1>. Values: 1-byte unsigned +integers represented in US-ASCII +
    • boolean: XML tag <boolean>. Values: "0" or "1" +
    • int: XML tags <i4> or <int>. Values: 4-byte +signed integers represented in US-ASCII. +
    • long: XML tag <ex:i8>. Values: 8-byte signed integers +represented in US-ASCII. +
    • float: XML tag <ex:float>. Values: Single precision +floating point numbers represented in US-ASCII. +
    • double: XML tag <double>. Values: Double precision +floating point numbers represented in US-ASCII. +
    • ustring: XML tag <;string>. Values: String values +represented as UTF-8. XML does not permit all Unicode characters in literal +data. In particular, NULLs and control chars are not allowed. Additionally, +XML processors are required to replace carriage returns with line feeds and to +replace CRLF sequences with line feeds. Programming languages that we work +with do not impose these restrictions on string types. To work around these +restrictions, disallowed characters and CRs are percent escaped in strings. +The '%' character is also percent escaped. +
    • buffer: XML tag <string&>. Values: Arbitrary binary +data. Represented as hexBinary, each byte is replaced by its 2-byte +hexadecimal representation. +
    + +Composite types are serialized as follows: + +
      +
    • class: XML tag <struct>. A struct is a sequence of +<member> elements. Each <member> element has a <name> +element and a <value> element. The <name> is a string that must +match /[a-zA-Z][a-zA-Z0-9_]*/. The value of the member is represented +by a <value> element. + +
    • vector: XML tag <array<. An <array> contains a +single <data> element. The <data> element is a sequence of +<value> elements each of which represents an element of the vector. + +
    • map: XML tag <array>. Same as vector. + +
    + +For example: + +
    
+class {
+  int           MY_INT;            // value 5
+  vector MY_VEC;            // values 0.1, -0.89, 2.45e4
+  buffer        MY_BUF;            // value '\00\n\tabc%'
+}
+
    + +is serialized as + +
    
+<value>
+  <struct>
+    <member>
+      <name>MY_INT</name>
+      <value><i4>5</i4></value>
+    </member>
+    <member>
+      <name>MY_VEC</name>
+      <value>
+        <array>
+          <data>
+            <value><ex:float>0.1</ex:float></value>
+            <value><ex:float>-0.89</ex:float></value>
+            <value><ex:float>2.45e4</ex:float></value>
+          </data>
+        </array>
+      </value>
+    </member>
+    <member>
+      <name>MY_BUF</name>
+      <value><string>%00\n\tabc%25</string></value>
+    </member>
+  </struct>
+</value> 
+
    ]]> +     +     + + + + + + + Avro.]]> + + + + + + + + + + + + + + + + + + + + + Avro.]]> + + + + + + + + + + + + Avro.]]> + + + + + + + + + + + + Avro.]]> + + + + + + + + + Avro.]]> + + + + + + + + + + + + Avro.]]> + + + + + + + + + + + + Avro.]]> + + + + + + + + + + + + + + + + + + + + + Avro.]]> + + + + + + + + + + + + Avro.]]> + + + + + + + + + + + + Avro.]]> + + + + + + + + + + + + Avro.]]> + + + + + + + + + + + + Avro.]]> + + + + + + + + + + + + Avro.]]> + + + + + + + + + + + + Avro.]]> + + + + + + + + + Avro.]]> + + + + + + + + + + + + Avro.]]> + + + + + + (DEPRECATED) This package contains classes needed for code generation + from the hadoop record compiler. CppGenerator and JavaGenerator + are the main entry points from the parser. There are classes + corrsponding to every primitive type and compound type + included in Hadoop record I/O syntax. +

    + +

    + DEPRECATED: Replaced by Avro. +

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This task takes the given record definition files and compiles them into + java or c++ + files. It is then up to the user to compile the generated files. + +

    The task requires the file or the nested fileset element to be + specified. Optional attributes are language (set the output + language, default is "java"), + destdir (name of the destination directory for generated java/c++ + code, default is ".") and failonerror (specifies error handling + behavior. default is true). +

    Usage

    + 
    + <recordcc
+       destdir="${basedir}/gensrc"
+       language="java">
+   <fileset include="**\/*.jr" />
+ </recordcc>
+
    + + @deprecated Replaced by Avro.]]> +     +     + +     + + + + + + ]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Avro.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Avro.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Avro.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Avro.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Avro.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Avro.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Avro.]]> + + + + + + (DEPRECATED) This package contains code generated by JavaCC from the + Hadoop record syntax file rcc.jj. For details about the + record file syntax please @see org.apache.hadoop.record. +

    + +

    + DEPRECATED: Replaced by Avro. +

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Avro.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Avro.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Avro.]]> + + + + + + + + + + + + + + + + + + + Avro.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Avro.]]> + + + + + + + + + + + + + + + + Avro.]]> + + + + + + + + + + + + + + + + + + + + + + Avro.]]> + + + + + + + + + + + + + + + + + + + + (cause==null ? null : cause.toString()) (which + typically contains the class and detail message of cause). + @param cause the cause (which is saved for later retrieval by the + {@link #getCause()} method). (A null value is + permitted, and indicates that the cause is nonexistent or + unknown.)]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + mapping + and mapping]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + /host@realm. + @param principalName principal name of format as described above + @return host name if the the string conforms to the above format, else null]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + "jack" + + @param userName + @return userName without login method]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + the return type of the run method + @param action the method to execute + @return the value from the run method]]> + + + + + + + + the return type of the run method + @param action the method to execute + @return the value from the run method + @throws IOException if the action throws an IOException + @throws Error if the action throws an Error + @throws RuntimeException if the action throws a RuntimeException + @throws InterruptedException if the action throws an InterruptedException + @throws UndeclaredThrowableException if the action throws something else]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + (cause==null ? null : cause.toString()) (which + typically contains the class and detail message of cause). + @param cause the cause (which is saved for later retrieval by the + {@link #getCause()} method). (A null value is + permitted, and indicates that the cause is nonexistent or + unknown.)]]> + + + + + + + + + + + + + + does not provide the stack trace for security purposes.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + A User-Agent String is considered to be a browser if it matches + any of the regex patterns from browser-useragent-regex; the default + behavior is to consider everything a browser that matches the following: + "^Mozilla.*,^Opera.*". Subclasses can optionally override + this method to use different behavior. + + @param userAgent The User-Agent String, or null if there isn't one + @return true if the User-Agent String refers to a browser, false if not]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The type of the token identifier]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + T extends TokenIdentifier]]> + + + + + + + + + + DelegationTokenAuthenticatedURL. +

    + An instance of the default {@link DelegationTokenAuthenticator} will be + used.]]> + + + + + DelegationTokenAuthenticatedURL. + + @param authenticator the {@link DelegationTokenAuthenticator} instance to + use, if null the default one will be used.]]> + + + + + DelegationTokenAuthenticatedURL using the default + {@link DelegationTokenAuthenticator} class. + + @param connConfigurator a connection configurator.]]> + + + + + DelegationTokenAuthenticatedURL. + + @param authenticator the {@link DelegationTokenAuthenticator} instance to + use, if null the default one will be used. + @param connConfigurator a connection configurator.]]> + + + + + + + + + + + + The default class is {@link KerberosDelegationTokenAuthenticator} + + @return the delegation token authenticator class to use as default.]]> + + + + + + + This method is provided to enable WebHDFS backwards compatibility. + + @param useQueryString TRUE if the token is transmitted in the + URL query string, FALSE if the delegation token is transmitted + using the {@link DelegationTokenAuthenticator#DELEGATION_TOKEN_HEADER} HTTP + header.]]> + + + + + TRUE if the token is transmitted in the URL query + string, FALSE if the delegation token is transmitted using the + {@link DelegationTokenAuthenticator#DELEGATION_TOKEN_HEADER} HTTP header.]]> + + + + + + + + + + + + + + + + + + Authenticator. + + @param url the URL to connect to. Only HTTP/S URLs are supported. + @param token the authentication token being used for the user. + @return an authenticated {@link HttpURLConnection}. + @throws IOException if an IO error occurred. + @throws AuthenticationException if an authentication exception occurred.]]> + + + + + + + + + + Authenticator. If the doAs parameter is not NULL, + the request will be done on behalf of the specified doAs user. + + @param url the URL to connect to. Only HTTP/S URLs are supported. + @param token the authentication token being used for the user. + @param doAs user to do the the request on behalf of, if NULL the request is + as self. + @return an authenticated {@link HttpURLConnection}. + @throws IOException if an IO error occurred. + @throws AuthenticationException if an authentication exception occurred.]]> + + + + + + + + + + Authenticator + for authentication. + + @param url the URL to get the delegation token from. Only HTTP/S URLs are + supported. + @param token the authentication token being used for the user where the + Delegation token will be stored. + @param renewer the renewer user. + @return a delegation token. + @throws IOException if an IO error occurred. + @throws AuthenticationException if an authentication exception occurred.]]> + + + + + + + + + + + Authenticator + for authentication. + + @param url the URL to get the delegation token from. Only HTTP/S URLs are + supported. + @param token the authentication token being used for the user where the + Delegation token will be stored. + @param renewer the renewer user. + @param doAsUser the user to do as, which will be the token owner. + @return a delegation token. + @throws IOException if an IO error occurred. + @throws AuthenticationException if an authentication exception occurred.]]> + + + + + + + + + Authenticator for authentication. + + @param url the URL to renew the delegation token from. Only HTTP/S URLs are + supported. + @param token the authentication token with the Delegation Token to renew. + @throws IOException if an IO error occurred. + @throws AuthenticationException if an authentication exception occurred.]]> + + + + + + + + + + Authenticator for authentication. + + @param url the URL to renew the delegation token from. Only HTTP/S URLs are + supported. + @param token the authentication token with the Delegation Token to renew. + @param doAsUser the user to do as, which will be the token owner. + @throws IOException if an IO error occurred. + @throws AuthenticationException if an authentication exception occurred.]]> + + + + + + + + Authenticator. + + @param url the URL to cancel the delegation token from. Only HTTP/S URLs + are supported. + @param token the authentication token with the Delegation Token to cancel. + @throws IOException if an IO error occurred.]]> + + + + + + + + + Authenticator. + + @param url the URL to cancel the delegation token from. Only HTTP/S URLs + are supported. + @param token the authentication token with the Delegation Token to cancel. + @param doAsUser the user to do as, which will be the token owner. + @throws IOException if an IO error occurred.]]> + + + + DelegationTokenAuthenticatedURL is a + {@link AuthenticatedURL} sub-class with built-in Hadoop Delegation Token + functionality. +

    + The authentication mechanisms supported by default are Hadoop Simple + authentication (also known as pseudo authentication) and Kerberos SPNEGO + authentication. +

    + Additional authentication mechanisms can be supported via {@link + DelegationTokenAuthenticator} implementations. +

    + The default {@link DelegationTokenAuthenticator} is the {@link + KerberosDelegationTokenAuthenticator} class which supports + automatic fallback from Kerberos SPNEGO to Hadoop Simple authentication via + the {@link PseudoDelegationTokenAuthenticator} class. +

    + AuthenticatedURL instances are not thread-safe.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Authenticator + for authentication. + + @param url the URL to get the delegation token from. Only HTTP/S URLs are + supported. + @param token the authentication token being used for the user where the + Delegation token will be stored. + @param renewer the renewer user. + @throws IOException if an IO error occurred. + @throws AuthenticationException if an authentication exception occurred.]]> + + + + + + + + + + + Authenticator + for authentication. + + @param url the URL to get the delegation token from. Only HTTP/S URLs are + supported. + @param token the authentication token being used for the user where the + Delegation token will be stored. + @param renewer the renewer user. + @param doAsUser the user to do as, which will be the token owner. + @throws IOException if an IO error occurred. + @throws AuthenticationException if an authentication exception occurred.]]> + + + + + + + + + + Authenticator for authentication. + + @param url the URL to renew the delegation token from. Only HTTP/S URLs are + supported. + @param token the authentication token with the Delegation Token to renew. + @throws IOException if an IO error occurred. + @throws AuthenticationException if an authentication exception occurred.]]> + + + + + + + + + + + Authenticator for authentication. + + @param url the URL to renew the delegation token from. Only HTTP/S URLs are + supported. + @param token the authentication token with the Delegation Token to renew. + @param doAsUser the user to do as, which will be the token owner. + @throws IOException if an IO error occurred. + @throws AuthenticationException if an authentication exception occurred.]]> + + + + + + + + + Authenticator. + + @param url the URL to cancel the delegation token from. Only HTTP/S URLs + are supported. + @param token the authentication token with the Delegation Token to cancel. + @throws IOException if an IO error occurred.]]> + + + + + + + + + + Authenticator. + + @param url the URL to cancel the delegation token from. Only HTTP/S URLs + are supported. + @param token the authentication token with the Delegation Token to cancel. + @param doAsUser the user to do as, which will be the token owner. + @throws IOException if an IO error occurred.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + KerberosDelegationTokenAuthenticator provides support for + Kerberos SPNEGO authentication mechanism and support for Hadoop Delegation + Token operations. +

    + It falls back to the {@link PseudoDelegationTokenAuthenticator} if the HTTP + endpoint does not trigger a SPNEGO authentication]]> + + + + + + + + + PseudoDelegationTokenAuthenticator provides support for + Hadoop's pseudo authentication mechanism that accepts + the user name specified as a query string parameter and support for Hadoop + Delegation Token operations. +

    + This mimics the model of Hadoop Simple authentication trusting the + {@link UserGroupInformation#getCurrentUser()} value.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + live. + @return a (snapshotted) map of blocker name->description values]]> + + + + + + + + + + + + + Do nothing if the service is null or not + in a state in which it can be/needs to be stopped. +

    + The service state is checked before the operation begins. + This process is not thread safe. + @param service a service or null]]> + + + + + + + + + + + + + + + + + + + + + + + + + + +

  • Any long-lived operation here will prevent the service state + change from completing in a timely manner.
    • +
  • If another thread is somehow invoked from the listener, and + that thread invokes the methods of the service (including + subclass-specific methods), there is a risk of a deadlock.
    • + + + + @param service the service that has changed.]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Clients and/or applications can use the provided Progressable + to explicitly report progress to the Hadoop framework. This is especially + important for operations which take significant amount of time since, + in-lieu of the reported progress, the framework has to assume that an error + has occured and time-out the operation.

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Class is to be obtained + @return the correctly typed Class of the given object.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + kill -0 command or equivalent]]> + + + + + + + + + + + + + + + + + + + ".cmd" on Windows, or ".sh" otherwise. + + @param parent File parent directory + @param basename String script file basename + @return File referencing the script in the directory]]> + + + + + + ".cmd" on Windows, or ".sh" otherwise. + + @param basename String script file basename + @return String script file name]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + IOException. + @return the path to {@link #WINUTILS_EXE} + @throws RuntimeException if the path is not resolvable]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Shell interface. + @param cmd shell command to execute. + @return the output of the executed command.]]> + + + + + + + + + Shell interface. + @param env the map of environment key=value + @param cmd shell command to execute. + @param timeout time in milliseconds after which script should be marked timeout + @return the output of the executed command. + @throws IOException on any problem.]]> + + + + + + + + Shell interface. + @param env the map of environment key=value + @param cmd shell command to execute. + @return the output of the executed command. + @throws IOException on any problem.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + CreateProcess synchronization object.]]> + + + + + os.name property.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Important: caller must check for this value being null. + The lack of such checks has led to many support issues being raised. +

    + @deprecated use one of the exception-raising getter methods, + specifically {@link #getWinUtilsPath()} or {@link #getWinUtilsFile()}]]> + + + + + + + + + + + + + + Shell can be used to run shell commands like du or + df. It also offers facilities to gate commands by + time-intervals.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Tool, is the standard for any Map-Reduce tool/application. + The tool/application should delegate the handling of + + standard command-line options to {@link ToolRunner#run(Tool, String[])} + and only handle its custom arguments.

    + +

    Here is how a typical Tool is implemented:

    + 

    +     public class MyApp extends Configured implements Tool {
+     
+       public int run(String[] args) throws Exception {
+         // Configuration processed by ToolRunner
+         Configuration conf = getConf();
+         
+         // Create a JobConf using the processed conf
+         JobConf job = new JobConf(conf, MyApp.class);
+         
+         // Process custom command-line options
+         Path in = new Path(args[1]);
+         Path out = new Path(args[2]);
+         
+         // Specify various job-specific parameters     
+         job.setJobName("my-app");
+         job.setInputPath(in);
+         job.setOutputPath(out);
+         job.setMapperClass(MyMapper.class);
+         job.setReducerClass(MyReducer.class);
+
+         // Submit the job, then poll for progress until the job is complete
+         RunningJob runningJob = JobClient.runJob(job);
+         if (runningJob.isSuccessful()) {
+           return 0;
+         } else {
+           return 1;
+         }
+       }
+       
+       public static void main(String[] args) throws Exception {
+         // Let ToolRunner handle generic command-line options 
+         int res = ToolRunner.run(new Configuration(), new MyApp(), args);
+         
+         System.exit(res);
+       }
+     }
+

    + + @see GenericOptionsParser + @see ToolRunner]]> +     + + + + + + + + + + + + + Tool by {@link Tool#run(String[])}, after + parsing with the given generic arguments. Uses the given + Configuration, or builds one if null. + + Sets the Tool's configuration with the possibly modified + version of the conf. + + @param conf Configuration for the Tool. + @param tool Tool to run. + @param args command-line arguments to the tool. + @return exit code of the {@link Tool#run(String[])} method.]]> + + + + + + + + Tool with its Configuration. + + Equivalent to run(tool.getConf(), tool, args). + + @param tool Tool to run. + @param args command-line arguments to the tool. + @return exit code of the {@link Tool#run(String[])} method.]]> + + + + + + + + + + + + + + + + + ToolRunner can be used to run classes implementing + Tool interface. It works in conjunction with + {@link GenericOptionsParser} to parse the + + generic hadoop command line arguments and modifies the + Configuration of the Tool. The + application-specific options are passed along without being modified. +

    + + @see Tool + @see GenericOptionsParser]]> +     +     + + + + +     + + + + + + + + + + + this filter. + @param nbHash The number of hash function to consider. + @param hashType type of the hashing function (see + {@link org.apache.hadoop.util.hash.Hash}).]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Bloom filter, as defined by Bloom in 1970. +

    + The Bloom filter is a data structure that was introduced in 1970 and that has been adopted by + the networking research community in the past decade thanks to the bandwidth efficiencies that it + offers for the transmission of set membership information between networked hosts. A sender encodes + the information into a bit vector, the Bloom filter, that is more compact than a conventional + representation. Computation and space costs for construction are linear in the number of elements. + The receiver uses the filter to test whether various elements are members of the set. Though the + filter will occasionally return a false positive, it will never return a false negative. When creating + the filter, the sender can choose its desired point in a trade-off between the false positive rate and the size. + +

    + Originally created by + European Commission One-Lab Project 034819. + + @see Filter The general behavior of a filter + + @see Space/Time Trade-Offs in Hash Coding with Allowable Errors]]> + + + + + + + + + + + + + this filter. + @param nbHash The number of hash function to consider. + @param hashType type of the hashing function (see + {@link org.apache.hadoop.util.hash.Hash}).]]> + + + + + + + + + this counting Bloom filter. +

    + Invariant: nothing happens if the specified key does not belong to this counter Bloom filter. + @param key The key to remove.]]> + + + + + + + + + + + + key -> count map. +

    NOTE: due to the bucket size of this filter, inserting the same + key more than 15 times will cause an overflow at all filter positions + associated with this key, and it will significantly increase the error + rate for this and other keys. For this reason the filter can only be + used to store small count values 0 <= N << 15. + @param key key to be tested + @return 0 if the key is not present. Otherwise, a positive value v will + be returned such that v == count with probability equal to the + error rate of this filter, and v > count otherwise. + Additionally, if the filter experienced an underflow as a result of + {@link #delete(Key)} operation, the return value may be lower than the + count with the probability of the false negative rate of such + filter.]]> + + + + + + + + + + + + + + + + + + + + + + counting Bloom filter, as defined by Fan et al. in a ToN + 2000 paper. +

    + A counting Bloom filter is an improvement to standard a Bloom filter as it + allows dynamic additions and deletions of set membership information. This + is achieved through the use of a counting vector instead of a bit vector. +

    + Originally created by + European Commission One-Lab Project 034819. + + @see Filter The general behavior of a filter + + @see Summary cache: a scalable wide-area web cache sharing protocol]]> + + + + + + + + + + + + + + Builds an empty Dynamic Bloom filter. + @param vectorSize The number of bits in the vector. + @param nbHash The number of hash function to consider. + @param hashType type of the hashing function (see + {@link org.apache.hadoop.util.hash.Hash}). + @param nr The threshold for the maximum number of keys to record in a + dynamic Bloom filter row.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + dynamic Bloom filter, as defined in the INFOCOM 2006 paper. +

    + A dynamic Bloom filter (DBF) makes use of a s * m bit matrix but + each of the s rows is a standard Bloom filter. The creation + process of a DBF is iterative. At the start, the DBF is a 1 * m + bit matrix, i.e., it is composed of a single standard Bloom filter. + It assumes that nr elements are recorded in the + initial bit vector, where nr <= n (n is + the cardinality of the set A to record in the filter). +

    + As the size of A grows during the execution of the application, + several keys must be inserted in the DBF. When inserting a key into the DBF, + one must first get an active Bloom filter in the matrix. A Bloom filter is + active when the number of recorded keys, nr, is + strictly less than the current cardinality of A, n. + If an active Bloom filter is found, the key is inserted and + nr is incremented by one. On the other hand, if there + is no active Bloom filter, a new one is created (i.e., a new row is added to + the matrix) according to the current size of A and the element + is added in this new Bloom filter and the nr value of + this new Bloom filter is set to one. A given key is said to belong to the + DBF if the k positions are set to one in one of the matrix rows. +

    + Originally created by + European Commission One-Lab Project 034819. + + @see Filter The general behavior of a filter + @see BloomFilter A Bloom filter + + @see Theory and Network Applications of Dynamic Bloom Filters]]> + + + + + + + + + Builds a hash function that must obey to a given maximum number of returned values and a highest value. + @param maxValue The maximum highest returned value. + @param nbHash The number of resulting hashed values. + @param hashType type of the hashing function (see {@link Hash}).]]> + + + + + this hash function. A NOOP]]> + + + + + + + + + + + + + + + + + + + The idea is to randomly select a bit to reset.]]> + + + + + + The idea is to select the bit to reset that will generate the minimum + number of false negative.]]> + + + + + + The idea is to select the bit to reset that will remove the maximum number + of false positive.]]> + + + + + + The idea is to select the bit to reset that will, at the same time, remove + the maximum number of false positve while minimizing the amount of false + negative generated.]]> + + + + + Originally created by + European Commission One-Lab Project 034819.]]> + + + + + + + + + + + + + + this filter. + @param nbHash The number of hash function to consider. + @param hashType type of the hashing function (see + {@link org.apache.hadoop.util.hash.Hash}).]]> + + + + + + + + + this retouched Bloom filter. +

    + Invariant: if the false positive is null, nothing happens. + @param key The false positive key to add.]]> + + + + + + this retouched Bloom filter. + @param coll The collection of false positive.]]> + + + + + + this retouched Bloom filter. + @param keys The list of false positive.]]> + + + + + + this retouched Bloom filter. + @param keys The array of false positive.]]> + + + + + + + this retouched Bloom filter. + @param scheme The selective clearing scheme to apply.]]> + + + + + + + + + + + + retouched Bloom filter, as defined in the CoNEXT 2006 paper. +

    + It allows the removal of selected false positives at the cost of introducing + random false negatives, and with the benefit of eliminating some random false + positives at the same time. + +

    + Originally created by + European Commission One-Lab Project 034819. + + @see Filter The general behavior of a filter + @see BloomFilter A Bloom filter + @see RemoveScheme The different selective clearing algorithms + + @see Retouched Bloom Filters: Allowing Networked Applications to Trade Off Selected False Positives Against False Negatives]]> + + + + + + + + + + diff --git a/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Common_2.8.0.xml b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Common_2.8.0.xml new file mode 100644 index 00000000000..453bd245492 --- /dev/null +++ b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Common_2.8.0.xml @@ -0,0 +1,113 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Core_2.8.0.xml b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Core_2.8.0.xml new file mode 100644 index 00000000000..4dbf6d691b8 --- /dev/null +++ b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_Core_2.8.0.xml @@ -0,0 +1,27490 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + FileStatus of a given cache file on hdfs + @throws IOException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + DistributedCache is a facility provided by the Map-Reduce + framework to cache files (text, archives, jars etc.) needed by applications. +

    + +

    Applications specify the files, via urls (hdfs:// or http://) to be cached + via the {@link org.apache.hadoop.mapred.JobConf}. The + DistributedCache assumes that the files specified via urls are + already present on the {@link FileSystem} at the path specified by the url + and are accessible by every machine in the cluster.

    + +

    The framework will copy the necessary files on to the slave node before + any tasks for the job are executed on that node. Its efficiency stems from + the fact that the files are only copied once per job and the ability to + cache archives which are un-archived on the slaves.

    + +

    DistributedCache can be used to distribute simple, read-only + data/text files and/or more complex types such as archives, jars etc. + Archives (zip, tar and tgz/tar.gz files) are un-archived at the slave nodes. + Jars may be optionally added to the classpath of the tasks, a rudimentary + software distribution mechanism. Files have execution permissions. + In older version of Hadoop Map/Reduce users could optionally ask for symlinks + to be created in the working directory of the child task. In the current + version symlinks are always created. If the URL does not have a fragment + the name of the file or directory will be used. If multiple files or + directories map to the same link name, the last one added, will be used. All + others will not even be downloaded.

    + +

    DistributedCache tracks modification timestamps of the cache + files. Clearly the cache files should not be modified by the application + or externally while the job is executing.

    + +

    Here is an illustrative example on how to use the + DistributedCache:

    + 

    +     // Setting up the cache for the application
+     
+     1. Copy the requisite files to the FileSystem:
+     
+     $ bin/hadoop fs -copyFromLocal lookup.dat /myapp/lookup.dat  
+     $ bin/hadoop fs -copyFromLocal map.zip /myapp/map.zip  
+     $ bin/hadoop fs -copyFromLocal mylib.jar /myapp/mylib.jar
+     $ bin/hadoop fs -copyFromLocal mytar.tar /myapp/mytar.tar
+     $ bin/hadoop fs -copyFromLocal mytgz.tgz /myapp/mytgz.tgz
+     $ bin/hadoop fs -copyFromLocal mytargz.tar.gz /myapp/mytargz.tar.gz
+     
+     2. Setup the application's JobConf:
+     
+     JobConf job = new JobConf();
+     DistributedCache.addCacheFile(new URI("/myapp/lookup.dat#lookup.dat"), 
+                                   job);
+     DistributedCache.addCacheArchive(new URI("/myapp/map.zip", job);
+     DistributedCache.addFileToClassPath(new Path("/myapp/mylib.jar"), job);
+     DistributedCache.addCacheArchive(new URI("/myapp/mytar.tar", job);
+     DistributedCache.addCacheArchive(new URI("/myapp/mytgz.tgz", job);
+     DistributedCache.addCacheArchive(new URI("/myapp/mytargz.tar.gz", job);
+     
+     3. Use the cached files in the {@link org.apache.hadoop.mapred.Mapper}
+     or {@link org.apache.hadoop.mapred.Reducer}:
+     
+     public static class MapClass extends MapReduceBase  
+     implements Mapper<K, V, K, V> {
+     
+       private Path[] localArchives;
+       private Path[] localFiles;
+       
+       public void configure(JobConf job) {
+         // Get the cached archives/files
+         File f = new File("./map.zip/some/file/in/zip.txt");
+       }
+       
+       public void map(K key, V value, 
+                       OutputCollector<K, V> output, Reporter reporter) 
+       throws IOException {
+         // Use data from the cached archives/files here
+         // ...
+         // ...
+         output.collect(k, v);
+       }
+     }
+     
+
    + + It is also very common to use the DistributedCache by using + {@link org.apache.hadoop.util.GenericOptionsParser}. + + This class includes methods that should be used by users + (specifically those mentioned in the example above, as well + as {@link DistributedCache#addArchiveToClassPath(Path, Configuration)}), + as well as methods intended for use by the MapReduce framework + (e.g., {@link org.apache.hadoop.mapred.JobClient}). + + @see org.apache.hadoop.mapred.JobConf + @see org.apache.hadoop.mapred.JobClient + @see org.apache.hadoop.mapreduce.Job]]> +     +     + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + JobTracker, + as {@link JobTracker.State} + + {@link JobTracker.State} should no longer be used on M/R 2.x. The function + is kept to be compatible with M/R 1.x applications. + + @return the invalid state of the JobTracker.]]> + + + + + + + + + + + + + + ClusterStatus provides clients with information such as: +
      +
    1. + Size of the cluster. +
      2. +
    2. + Name of the trackers. +
      3. +
    3. + Task capacity of the cluster. +
      4. +
    4. + The number of currently running map and reduce tasks. +
      5. +
    5. + State of the JobTracker. +
      6. +
    6. + Details regarding black listed trackers. +
      7. +
    + +

    Clients can query for the latest ClusterStatus, via + {@link JobClient#getClusterStatus()}.

    + + @see JobClient]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Counters represent global counters, defined either by the + Map-Reduce framework or applications. Each Counter can be of + any {@link Enum} type.

    + +

    Counters are bunched into {@link Group}s, each comprising of + counters from a particular Enum class.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Group of counters, comprising of counters from a particular + counter {@link Enum} class. + +

    Grouphandles localization of the class name and the + counter names.

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + FileInputFormat always returns + true. Implementations that may deal with non-splittable files must + override this method. + + FileInputFormat implementations can override this and return + false to ensure that individual input files are never split-up + so that {@link Mapper}s process entire files. + + @param fs the file system that the file is on + @param filename the file name to check + @return is this file splitable?]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + FileInputFormat is the base class for all file-based + InputFormats. This provides a generic implementation of + {@link #getSplits(JobConf, int)}. + + Implementations of FileInputFormat can also override the + {@link #isSplitable(FileSystem, Path)} method to prevent input files + from being split-up in certain situations. Implementations that may + deal with non-splittable files must override this method, since + the default implementation assumes splitting is always possible.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if the job output should be compressed, + false otherwise]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Tasks' Side-Effect Files + +

    Note: The following is valid only if the {@link OutputCommitter} + is {@link FileOutputCommitter}. If OutputCommitter is not + a FileOutputCommitter, the task's temporary output + directory is same as {@link #getOutputPath(JobConf)} i.e. + ${mapreduce.output.fileoutputformat.outputdir}$

    + +

    Some applications need to create/write-to side-files, which differ from + the actual job-outputs. + +

    In such cases there could be issues with 2 instances of the same TIP + (running simultaneously e.g. speculative tasks) trying to open/write-to the + same file (path) on HDFS. Hence the application-writer will have to pick + unique names per task-attempt (e.g. using the attemptid, say + attempt_200709221812_0001_m_000000_0), not just per TIP.

    + +

    To get around this the Map-Reduce framework helps the application-writer + out by maintaining a special + ${mapreduce.output.fileoutputformat.outputdir}/_temporary/_${taskid} + sub-directory for each task-attempt on HDFS where the output of the + task-attempt goes. On successful completion of the task-attempt the files + in the ${mapreduce.output.fileoutputformat.outputdir}/_temporary/_${taskid} (only) + are promoted to ${mapreduce.output.fileoutputformat.outputdir}. Of course, the + framework discards the sub-directory of unsuccessful task-attempts. This + is completely transparent to the application.

    + +

    The application-writer can take advantage of this by creating any + side-files required in ${mapreduce.task.output.dir} during execution + of his reduce-task i.e. via {@link #getWorkOutputPath(JobConf)}, and the + framework will move them out similarly - thus she doesn't have to pick + unique paths per task-attempt.

    + +

    Note: the value of ${mapreduce.task.output.dir} during + execution of a particular task-attempt is actually + ${mapreduce.output.fileoutputformat.outputdir}/_temporary/_{$taskid}, and this value is + set by the map-reduce framework. So, just create any side-files in the + path returned by {@link #getWorkOutputPath(JobConf)} from map/reduce + task to take advantage of this feature.

    + +

    The entire discussion holds true for maps of jobs with + reducer=NONE (i.e. 0 reduces) since output of the map, in that case, + goes directly to HDFS.

    + + @return the {@link Path} to the task's temporary output directory + for the map-reduce job.]]> +     +     + + + + + + + + + + + + + The generated name can be used to create custom files from within the + different tasks for the job, the names for different tasks will not collide + with each other.

    + +

    The given name is postfixed with the task type, 'm' for maps, 'r' for + reduces and the task partition number. For example, give a name 'test' + running on the first map o the job the generated name will be + 'test-m-00000'.

    + + @param conf the configuration for the job. + @param name the name to make unique. + @return a unique name accross all tasks of the job.]]> +     +     + + + + + The path can be used to create custom files from within the map and + reduce tasks. The path name will be unique for each task. The path parent + will be the job output directory.

    ls + +

    This method uses the {@link #getUniqueName} method to make the file name + unique for the task.

    + + @param conf the configuration for the job. + @param name the name for the file. + @return a unique path accross all tasks of the job.]]> +     +     + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    or + conf.setInt(FixedLengthInputFormat.FIXED_RECORD_LENGTH, recordLength); +

    + @see FixedLengthRecordReader]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + Each {@link InputSplit} is then assigned to an individual {@link Mapper} + for processing.

    + +

    Note: The split is a logical split of the inputs and the + input files are not physically split into chunks. For e.g. a split could + be <input-file-path, start, offset> tuple. + + @param job job configuration. + @param numSplits the desired number of splits, a hint. + @return an array of {@link InputSplit}s for the job.]]> + + + + + + + + + It is the responsibility of the RecordReader to respect + record boundaries while processing the logical split to present a + record-oriented view to the individual task.

    + + @param split the {@link InputSplit} + @param job the job that this split belongs to + @return a {@link RecordReader}]]> +     +     + + InputFormat describes the input-specification for a + Map-Reduce job. + +

    The Map-Reduce framework relies on the InputFormat of the + job to:

    +

      +
    1. + Validate the input-specification of the job. +
    2. + Split-up the input file(s) into logical {@link InputSplit}s, each of + which is then assigned to an individual {@link Mapper}. +
      3. +
    3. + Provide the {@link RecordReader} implementation to be used to glean + input records from the logical InputSplit for processing by + the {@link Mapper}. +
      4. +
    + +

    The default behavior of file-based {@link InputFormat}s, typically + sub-classes of {@link FileInputFormat}, is to split the + input into logical {@link InputSplit}s based on the total size, in + bytes, of the input files. However, the {@link FileSystem} blocksize of + the input files is treated as an upper bound for input splits. A lower bound + on the split size can be set via + + mapreduce.input.fileinputformat.split.minsize.

    + +

    Clearly, logical splits based on input-size is insufficient for many + applications since record boundaries are to be respected. In such cases, the + application has to also implement a {@link RecordReader} on whom lies the + responsibilty to respect record-boundaries and present a record-oriented + view of the logical InputSplit to the individual task. + + @see InputSplit + @see RecordReader + @see JobClient + @see FileInputFormat]]> + + + + + + + + + + InputSplit. + + @return the number of bytes in the input split. + @throws IOException]]> + + + + + + InputSplit is + located as an array of Strings. + @throws IOException]]> + + + + InputSplit represents the data to be processed by an + individual {@link Mapper}. + +

    Typically, it presents a byte-oriented view on the input and is the + responsibility of {@link RecordReader} of the job to process this and present + a record-oriented view. + + @see InputFormat + @see RecordReader]]> + + + + + + + + + + SplitLocationInfos describing how the split + data is stored at each location. A null value indicates that all the + locations have the data stored on disk. + @throws IOException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + JobClient.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + jobid doesn't correspond to any known job. + @throws IOException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + JobClient is the primary interface for the user-job to interact + with the cluster. + + JobClient provides facilities to submit jobs, track their + progress, access component-tasks' reports/logs, get the Map-Reduce cluster + status information etc. + +

    The job submission process involves: +

      +
    1. + Checking the input and output specifications of the job. +
      2. +
    2. + Computing the {@link InputSplit}s for the job. +
      3. +
    3. + Setup the requisite accounting information for the {@link DistributedCache} + of the job, if necessary. +
      4. +
    4. + Copying the job's jar and configuration to the map-reduce system directory + on the distributed file-system. +
      5. +
    5. + Submitting the job to the cluster and optionally monitoring + it's status. +
      6. +
    + + Normally the user creates the application, describes various facets of the + job via {@link JobConf} and then uses the JobClient to submit + the job and monitor its progress. + +

    Here is an example on how to use JobClient:

    + 

    +     // Create a new JobConf
+     JobConf job = new JobConf(new Configuration(), MyJob.class);
+     
+     // Specify various job-specific parameters     
+     job.setJobName("myjob");
+     
+     job.setInputPath(new Path("in"));
+     job.setOutputPath(new Path("out"));
+     
+     job.setMapperClass(MyJob.MyMapper.class);
+     job.setReducerClass(MyJob.MyReducer.class);
+
+     // Submit the job, then poll for progress until the job is complete
+     JobClient.runJob(job);
+
    + + Job Control + +

    At times clients would chain map-reduce jobs to accomplish complex tasks + which cannot be done via a single map-reduce job. This is fairly easy since + the output of the job, typically, goes to distributed file-system and that + can be used as the input for the next job.

    + +

    However, this also means that the onus on ensuring jobs are complete + (success/failure) lies squarely on the clients. In such situations the + various job-control options are: +

      +
    1. + {@link #runJob(JobConf)} : submits the job and returns only after + the job has completed. +
      2. +
    2. + {@link #submitJob(JobConf)} : only submits the job, then poll the + returned handle to the {@link RunningJob} to query status and make + scheduling decisions. +
      3. +
    3. + {@link JobConf#setJobEndNotificationURI(String)} : setup a notification + on job-completion, thus avoiding polling. +
      4. +
    + + @see JobConf + @see ClusterStatus + @see Tool + @see DistributedCache]]> +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + If the parameter {@code loadDefaults} is false, the new instance + will not load resources from the default files. + + @param loadDefaults specifies whether to load from the default files]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if framework should keep the intermediate files + for failed tasks, false otherwise.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if the outputs of the maps are to be compressed, + false otherwise.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This comparator should be provided if the equivalence rules for keys + for sorting the intermediates are different from those for grouping keys + before each call to + {@link Reducer#reduce(Object, java.util.Iterator, OutputCollector, Reporter)}.

    + +

    For key-value pairs (K1,V1) and (K2,V2), the values (V1, V2) are passed + in a single call to the reduce function if K1 and K2 compare as equal.

    + +

    Since {@link #setOutputKeyComparatorClass(Class)} can be used to control + how keys are sorted, this can be used in conjunction to simulate + secondary sort on values.

    + +

    Note: This is not a guarantee of the combiner sort being + stable in any sense. (In any case, with the order of available + map-outputs to the combiner being non-deterministic, it wouldn't make + that much sense.)

    + + @param theClass the comparator class to be used for grouping keys for the + combiner. It should implement RawComparator. + @see #setOutputKeyComparatorClass(Class)]]> +     +     + + + + This comparator should be provided if the equivalence rules for keys + for sorting the intermediates are different from those for grouping keys + before each call to + {@link Reducer#reduce(Object, java.util.Iterator, OutputCollector, Reporter)}.

    + +

    For key-value pairs (K1,V1) and (K2,V2), the values (V1, V2) are passed + in a single call to the reduce function if K1 and K2 compare as equal.

    + +

    Since {@link #setOutputKeyComparatorClass(Class)} can be used to control + how keys are sorted, this can be used in conjunction to simulate + secondary sort on values.

    + +

    Note: This is not a guarantee of the reduce sort being + stable in any sense. (In any case, with the order of available + map-outputs to the reduce being non-deterministic, it wouldn't make + that much sense.)

    + + @param theClass the comparator class to be used for grouping keys. + It should implement RawComparator. + @see #setOutputKeyComparatorClass(Class) + @see #setCombinerKeyGroupingComparator(Class)]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + combiner class used to combine map-outputs + before being sent to the reducers. Typically the combiner is same as the + the {@link Reducer} for the job i.e. {@link #getReducerClass()}. + + @return the user-defined combiner class used to combine map-outputs.]]> + + + + + + combiner class used to combine map-outputs + before being sent to the reducers. + +

    The combiner is an application-specified aggregation operation, which + can help cut down the amount of data transferred between the + {@link Mapper} and the {@link Reducer}, leading to better performance.

    + +

    The framework may invoke the combiner 0, 1, or multiple times, in both + the mapper and reducer tasks. In general, the combiner is called as the + sort/merge result is written to disk. The combiner must: +

      +
    • be side-effect free
      • +
    • have the same input and output key types and the same input and + output value types
      • +
    + +

    Typically the combiner is same as the Reducer for the + job i.e. {@link #setReducerClass(Class)}.

    + + @param theClass the user-defined combiner class used to combine + map-outputs.]]> +     +     + + + true. + + @return true if speculative execution be used for this job, + false otherwise.]]> + + + + + + true if speculative execution + should be turned on, else false.]]> + + + + + true. + + @return true if speculative execution be + used for this job for map tasks, + false otherwise.]]> + + + + + + true if speculative execution + should be turned on for map tasks, + else false.]]> + + + + + true. + + @return true if speculative execution be used + for reduce tasks for this job, + false otherwise.]]> + + + + + + true if speculative execution + should be turned on for reduce tasks, + else false.]]> + + + + + 1. + + @return the number of reduce tasks for this job.]]> + + + + + + Note: This is only a hint to the framework. The actual + number of spawned map tasks depends on the number of {@link InputSplit}s + generated by the job's {@link InputFormat#getSplits(JobConf, int)}. + + A custom {@link InputFormat} is typically used to accurately control + the number of map tasks for the job.

    + + How many maps? + +

    The number of maps is usually driven by the total size of the inputs + i.e. total number of blocks of the input files.

    + +

    The right level of parallelism for maps seems to be around 10-100 maps + per-node, although it has been set up to 300 or so for very cpu-light map + tasks. Task setup takes awhile, so it is best if the maps take at least a + minute to execute.

    + +

    The default behavior of file-based {@link InputFormat}s is to split the + input into logical {@link InputSplit}s based on the total size, in + bytes, of input files. However, the {@link FileSystem} blocksize of the + input files is treated as an upper bound for input splits. A lower bound + on the split size can be set via + + mapreduce.input.fileinputformat.split.minsize.

    + +

    Thus, if you expect 10TB of input data and have a blocksize of 128MB, + you'll end up with 82,000 maps, unless {@link #setNumMapTasks(int)} is + used to set it even higher.

    + + @param n the number of map tasks for this job. + @see InputFormat#getSplits(JobConf, int) + @see FileInputFormat + @see FileSystem#getDefaultBlockSize() + @see FileStatus#getBlockSize()]]> +     +     + + + 1. + + @return the number of reduce tasks for this job.]]> + + + + + + How many reduces? + +

    The right number of reduces seems to be 0.95 or + 1.75 multiplied by ( + available memory for reduce tasks + (The value of this should be smaller than + numNodes * yarn.nodemanager.resource.memory-mb + since the resource of memory is shared by map tasks and other + applications) / + + mapreduce.reduce.memory.mb). +

    + +

    With 0.95 all of the reduces can launch immediately and + start transfering map outputs as the maps finish. With 1.75 + the faster nodes will finish their first round of reduces and launch a + second wave of reduces doing a much better job of load balancing.

    + +

    Increasing the number of reduces increases the framework overhead, but + increases load balancing and lowers the cost of failures.

    + +

    The scaling factors above are slightly less than whole numbers to + reserve a few reduce slots in the framework for speculative-tasks, failures + etc.

    + + Reducer NONE + +

    It is legal to set the number of reduce-tasks to zero.

    + +

    In this case the output of the map-tasks directly go to distributed + file-system, to the path set by + {@link FileOutputFormat#setOutputPath(JobConf, Path)}. Also, the + framework doesn't sort the map-outputs before writing it out to HDFS.

    + + @param n the number of reduce tasks for this job.]]> +     +     + + + mapreduce.map.maxattempts + property. If this property is not already set, the default is 4 attempts. + + @return the max number of attempts per map task.]]> + + + + + + + + + + + mapreduce.reduce.maxattempts + property. If this property is not already set, the default is 4 attempts. + + @return the max number of attempts per reduce task.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + noFailures, the + tasktracker is blacklisted for this job. + + @param noFailures maximum no. of failures of a given job per tasktracker.]]> + + + + + blacklisted for this job. + + @return the maximum no. of failures of a given job per tasktracker.]]> + + + + + failed. + + Defaults to zero, i.e. any failed map-task results in + the job being declared as {@link JobStatus#FAILED}. + + @return the maximum percentage of map tasks that can fail without + the job being aborted.]]> + + + + + + failed. + + @param percent the maximum percentage of map tasks that can fail without + the job being aborted.]]> + + + + + failed. + + Defaults to zero, i.e. any failed reduce-task results + in the job being declared as {@link JobStatus#FAILED}. + + @return the maximum percentage of reduce tasks that can fail without + the job being aborted.]]> + + + + + + failed. + + @param percent the maximum percentage of reduce tasks that can fail without + the job being aborted.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The debug script can aid debugging of failed map tasks. The script is + given task's stdout, stderr, syslog, jobconf files as arguments.

    + +

    The debug command, run on the node where the map failed, is:

    + 

    + $script $stdout $stderr $syslog $jobconf.
+
    + +

    The script file is distributed through {@link DistributedCache} + APIs. The script needs to be symlinked.

    + +

    Here is an example on how to submit a script + 

    + job.setMapDebugScript("./myscript");
+ DistributedCache.createSymlink(job);
+ DistributedCache.addCacheFile("/debug/scripts/myscript#myscript");
+
    + + @param mDbgScript the script name]]> +     +     + + + + + + + + + The debug script can aid debugging of failed reduce tasks. The script + is given task's stdout, stderr, syslog, jobconf files as arguments.

    + +

    The debug command, run on the node where the map failed, is:

    + 

    + $script $stdout $stderr $syslog $jobconf.
+
    + +

    The script file is distributed through {@link DistributedCache} + APIs. The script file needs to be symlinked

    + +

    Here is an example on how to submit a script + 

    + job.setReduceDebugScript("./myscript");
+ DistributedCache.createSymlink(job);
+ DistributedCache.addCacheFile("/debug/scripts/myscript#myscript");
+
    + + @param rDbgScript the script name]]> +     +     + + + + + + + + null if it hasn't + been set. + @see #setJobEndNotificationURI(String)]]> + + + + + + The uri can contain 2 special parameters: $jobId and + $jobStatus. Those, if present, are replaced by the job's + identifier and completion-status respectively.

    + +

    This is typically used by application-writers to implement chaining of + Map-Reduce jobs in an asynchronous manner.

    + + @param uri the job end notification uri + @see JobStatus]]> +     +     + + + + When a job starts, a shared directory is created at location + + ${mapreduce.cluster.local.dir}/taskTracker/$user/jobcache/$jobid/work/ . + This directory is exposed to the users through + mapreduce.job.local.dir . + So, the tasks can use this space + as scratch space and share files among them.

    + This value is available as System property also. + + @return The localized job specific shared directory]]> +     +     + + + + For backward compatibility, if the job configuration sets the + key {@link #MAPRED_TASK_MAXVMEM_PROPERTY} to a value different + from {@link #DISABLED_MEMORY_LIMIT}, that value will be used + after converting it from bytes to MB. + @return memory required to run a map task of the job, in MB,]]> + + + + + + + + + For backward compatibility, if the job configuration sets the + key {@link #MAPRED_TASK_MAXVMEM_PROPERTY} to a value different + from {@link #DISABLED_MEMORY_LIMIT}, that value will be used + after converting it from bytes to MB. + @return memory required to run a reduce task of the job, in MB.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This method is deprecated. Now, different memory limits can be + set for map and reduce tasks of a job, in MB. +

    + For backward compatibility, if the job configuration sets the + key {@link #MAPRED_TASK_MAXVMEM_PROPERTY}, that value is returned. + Otherwise, this method will return the larger of the values returned by + {@link #getMemoryForMapTask()} and {@link #getMemoryForReduceTask()} + after converting them into bytes. + + @return Memory required to run a task of this job, in bytes. + @see #setMaxVirtualMemoryForTask(long) + @deprecated Use {@link #getMemoryForMapTask()} and + {@link #getMemoryForReduceTask()}]]> + + + + + + + mapred.task.maxvmem is split into + mapreduce.map.memory.mb + and mapreduce.map.memory.mb,mapred + each of the new key are set + as mapred.task.maxvmem / 1024 + as new values are in MB + + @param vmem Maximum amount of virtual memory in bytes any task of this job + can use. + @see #getMaxVirtualMemoryForTask() + @deprecated + Use {@link #setMemoryForMapTask(long mem)} and + Use {@link #setMemoryForReduceTask(long mem)}]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + k1=v1,k2=v2. Further it can + reference existing environment variables via $key on + Linux or %key% on Windows. + + Example: +

      +
    • A=foo - This will set the env variable A to foo.
      • +
    • B=$X:c This is inherit tasktracker's X env variable on Linux.
      • +
    • B=%X%;c This is inherit tasktracker's X env variable on Windows.
      • +
    + + @deprecated Use {@link #MAPRED_MAP_TASK_ENV} or + {@link #MAPRED_REDUCE_TASK_ENV}]]> +     + + + + k1=v1,k2=v2. Further it can + reference existing environment variables via $key on + Linux or %key% on Windows. + + Example: +
      +
    • A=foo - This will set the env variable A to foo.
      • +
    • B=$X:c This is inherit tasktracker's X env variable on Linux.
      • +
    • B=%X%;c This is inherit tasktracker's X env variable on Windows.
      • +
    ]]> +     +     + + + k1=v1,k2=v2. Further it can + reference existing environment variables via $key on + Linux or %key% on Windows. + + Example: +
      +
    • A=foo - This will set the env variable A to foo.
      • +
    • B=$X:c This is inherit tasktracker's X env variable on Linux.
      • +
    • B=%X%;c This is inherit tasktracker's X env variable on Windows.
      • +
    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + JobConf is the primary interface for a user to describe a + map-reduce job to the Hadoop framework for execution. The framework tries to + faithfully execute the job as-is described by JobConf, however: +
      +
    1. + Some configuration parameters might have been marked as + + final by administrators and hence cannot be altered. +
      2. +
    2. + While some job parameters are straight-forward to set + (e.g. {@link #setNumReduceTasks(int)}), some parameters interact subtly + with the rest of the framework and/or job-configuration and is relatively + more complex for the user to control finely + (e.g. {@link #setNumMapTasks(int)}). +
      3. +
    + +

    JobConf typically specifies the {@link Mapper}, combiner + (if any), {@link Partitioner}, {@link Reducer}, {@link InputFormat} and + {@link OutputFormat} implementations to be used etc. + +

    Optionally JobConf is used to specify other advanced facets + of the job such as Comparators to be used, files to be put in + the {@link DistributedCache}, whether or not intermediate and/or job outputs + are to be compressed (and how), debugability via user-provided scripts + ( {@link #setMapDebugScript(String)}/{@link #setReduceDebugScript(String)}), + for doing post-processing on task logs, task's stdout, stderr, syslog. + and etc.

    + +

    Here is an example on how to configure a job via JobConf:

    + 

    +     // Create a new JobConf
+     JobConf job = new JobConf(new Configuration(), MyJob.class);
+     
+     // Specify various job-specific parameters     
+     job.setJobName("myjob");
+     
+     FileInputFormat.setInputPaths(job, new Path("in"));
+     FileOutputFormat.setOutputPath(job, new Path("out"));
+     
+     job.setMapperClass(MyJob.MyMapper.class);
+     job.setCombinerClass(MyJob.MyReducer.class);
+     job.setReducerClass(MyJob.MyReducer.class);
+     
+     job.setInputFormat(SequenceFileInputFormat.class);
+     job.setOutputFormat(SequenceFileOutputFormat.class);
+
    + + @see JobClient + @see ClusterStatus + @see Tool + @see DistributedCache]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + any job + run on the jobtracker started at 200707121733, we would use : + 
     
+ JobID.getTaskIDsPattern("200707121733", null);
+
    + which will return : + 
     "job_200707121733_[0-9]*"
    + @param jtIdentifier jobTracker identifier, or null + @param jobId job number, or null + @return a regex pattern matching JobIDs]]> +     +     + + + An example JobID is : + job_200707121733_0003 , which represents the third job + running at the jobtracker started at 200707121733. +

    + Applications should never construct or parse JobID strings, but rather + use appropriate constructors or {@link #forName(String)} method. + + @see TaskID + @see TaskAttemptID]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Output pairs need not be of the same types as input pairs. A given + input pair may map to zero or many output pairs. Output pairs are + collected with calls to + {@link OutputCollector#collect(Object,Object)}.

    + +

    Applications can use the {@link Reporter} provided to report progress + or just indicate that they are alive. In scenarios where the application + takes significant amount of time to process individual key/value + pairs, this is crucial since the framework might assume that the task has + timed-out and kill that task. The other way of avoiding this is to set + + mapreduce.task.timeout to a high-enough value (or even zero for no + time-outs).

    + + @param key the input key. + @param value the input value. + @param output collects mapped keys and values. + @param reporter facility to report progress.]]> +     + + + Maps are the individual tasks which transform input records into a + intermediate records. The transformed intermediate records need not be of + the same type as the input records. A given input pair may map to zero or + many output pairs.

    + +

    The Hadoop Map-Reduce framework spawns one map task for each + {@link InputSplit} generated by the {@link InputFormat} for the job. + Mapper implementations can access the {@link JobConf} for the + job via the {@link JobConfigurable#configure(JobConf)} and initialize + themselves. Similarly they can use the {@link Closeable#close()} method for + de-initialization.

    + +

    The framework then calls + {@link #map(Object, Object, OutputCollector, Reporter)} + for each key/value pair in the InputSplit for that task.

    + +

    All intermediate values associated with a given output key are + subsequently grouped by the framework, and passed to a {@link Reducer} to + determine the final output. Users can control the grouping by specifying + a Comparator via + {@link JobConf#setOutputKeyComparatorClass(Class)}.

    + +

    The grouped Mapper outputs are partitioned per + Reducer. Users can control which keys (and hence records) go to + which Reducer by implementing a custom {@link Partitioner}. + +

    Users can optionally specify a combiner, via + {@link JobConf#setCombinerClass(Class)}, to perform local aggregation of the + intermediate outputs, which helps to cut down the amount of data transferred + from the Mapper to the Reducer. + +

    The intermediate, grouped outputs are always stored in + {@link SequenceFile}s. Applications can specify if and how the intermediate + outputs are to be compressed and which {@link CompressionCodec}s are to be + used via the JobConf.

    + +

    If the job has + zero + reduces then the output of the Mapper is directly written + to the {@link FileSystem} without grouping by keys.

    + +

    Example:

    + 

    +     public class MyMapper<K extends WritableComparable, V extends Writable> 
+     extends MapReduceBase implements Mapper<K, V, K, V> {
+     
+       static enum MyCounters { NUM_RECORDS }
+       
+       private String mapTaskId;
+       private String inputFile;
+       private int noRecords = 0;
+       
+       public void configure(JobConf job) {
+         mapTaskId = job.get(JobContext.TASK_ATTEMPT_ID);
+         inputFile = job.get(JobContext.MAP_INPUT_FILE);
+       }
+       
+       public void map(K key, V val,
+                       OutputCollector<K, V> output, Reporter reporter)
+       throws IOException {
+         // Process the <key, value> pair (assume this takes a while)
+         // ...
+         // ...
+         
+         // Let the framework know that we are alive, and kicking!
+         // reporter.progress();
+         
+         // Process some more
+         // ...
+         // ...
+         
+         // Increment the no. of <key, value> pairs processed
+         ++noRecords;
+
+         // Increment counters
+         reporter.incrCounter(NUM_RECORDS, 1);
+        
+         // Every 100 records update application-level status
+         if ((noRecords%100) == 0) {
+           reporter.setStatus(mapTaskId + " processed " + noRecords + 
+                              " from input-file: " + inputFile); 
+         }
+         
+         // Output the result
+         output.collect(key, val);
+       }
+     }
+
    + +

    Applications may write a custom {@link MapRunnable} to exert greater + control on map processing e.g. multi-threaded Mappers etc.

    + + @see JobConf + @see InputFormat + @see Partitioner + @see Reducer + @see MapReduceBase + @see MapRunnable + @see SequenceFile]]> +     +     + + + + + + + + + + + + + + + + + + + + + Provides default no-op implementations for a few methods, most non-trivial + applications need to override some of them.

    ]]> +     +     + + + + + + + + + + + <key, value> pairs. + +

    Mapping of input records to output records is complete when this method + returns.

    + + @param input the {@link RecordReader} to read the input records. + @param output the {@link OutputCollector} to collect the outputrecords. + @param reporter {@link Reporter} to report progress, status-updates etc. + @throws IOException]]> +     +     + + Custom implementations of MapRunnable can exert greater + control on map processing e.g. multi-threaded, asynchronous mappers etc.

    + + @see Mapper]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + nearly + equal content length.
    + Subclasses implement {@link #getRecordReader(InputSplit, JobConf, Reporter)} + to construct RecordReader's for MultiFileSplit's. + @see MultiFileSplit]]> +     +     + + + + + + + + + + + + + MultiFileSplit can be used to implement {@link RecordReader}'s, with + reading one record per file. + @see FileSplit + @see MultiFileInputFormat]]> + + + + + + + + + + + + + + + <key, value> pairs output by {@link Mapper}s + and {@link Reducer}s. + +

    OutputCollector is the generalization of the facility + provided by the Map-Reduce framework to collect data output by either the + Mapper or the Reducer i.e. intermediate outputs + or the output of the job.

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if task output recovery is supported, + false otherwise + @throws IOException + @see #recoverTask(TaskAttemptContext)]]> + + + + + + + true repeatable job commit is supported, + false otherwise + @throws IOException]]> + + + + + + + + + + + OutputCommitter. This is called from the application master + process, but it is called individually for each task. + + If an exception is thrown the task will be attempted again. + + @param taskContext Context of the task whose output is being recovered + @throws IOException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + OutputCommitter describes the commit of task output for a + Map-Reduce job. + +

    The Map-Reduce framework relies on the OutputCommitter of + the job to:

    +

      +
    1. + Setup the job during initialization. For example, create the temporary + output directory for the job during the initialization of the job. +
      2. +
    2. + Cleanup the job after the job completion. For example, remove the + temporary output directory after the job completion. +
      3. +
    3. + Setup the task temporary output. +
      4. +
    4. + Check whether a task needs a commit. This is to avoid the commit + procedure if a task does not need commit. +
      5. +
    5. + Commit of the task output. +
      6. +
    6. + Discard the task commit. +
      7. +
    + The methods in this class can be called from several different processes and + from several different contexts. It is important to know which process and + which context each is called from. Each method should be marked accordingly + in its documentation. It is also important to note that not all methods are + guaranteed to be called once and only once. If a method is not guaranteed to + have this property the output committer needs to handle this appropriately. + Also note it will only be in rare situations where they may be called + multiple times for the same task. + + @see FileOutputCommitter + @see JobContext + @see TaskAttemptContext]]> +     +     + + + + + + + + + + + + + + + + + + + This is to validate the output specification for the job when it is + a job is submitted. Typically checks that it does not already exist, + throwing an exception when it already exists, so that output is not + overwritten.

    + + @param ignored + @param job job configuration. + @throws IOException when output should not be attempted]]> +     +     + + OutputFormat describes the output-specification for a + Map-Reduce job. + +

    The Map-Reduce framework relies on the OutputFormat of the + job to:

    +

      +
    1. + Validate the output-specification of the job. For e.g. check that the + output directory doesn't already exist. +
    2. + Provide the {@link RecordWriter} implementation to be used to write out + the output files of the job. Output files are stored in a + {@link FileSystem}. +
      3. +
    + + @see RecordWriter + @see JobConf]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + Typically a hash function on a all or a subset of the key.

    + + @param key the key to be paritioned. + @param value the entry value. + @param numPartitions the total number of partitions. + @return the partition number for the key.]]> +     +     + + Partitioner controls the partitioning of the keys of the + intermediate map-outputs. The key (or a subset of the key) is used to derive + the partition, typically by a hash function. The total number of partitions + is the same as the number of reduce tasks for the job. Hence this controls + which of the m reduce tasks the intermediate key (and hence the + record) is sent for reduction.

    + + @see Reducer]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0.0 to 1.0. + @throws IOException]]> + + + + RecordReader reads <key, value> pairs from an + {@link InputSplit}. + +

    RecordReader, typically, converts the byte-oriented view of + the input, provided by the InputSplit, and presents a + record-oriented view for the {@link Mapper} and {@link Reducer} tasks for + processing. It thus assumes the responsibility of processing record + boundaries and presenting the tasks with keys and values.

    + + @see InputSplit + @see InputFormat]]> +     +     + + + + + + + + + + + + + + + + RecordWriter to future operations. + + @param reporter facility to report progress. + @throws IOException]]> + + + + RecordWriter writes the output <key, value> pairs + to an output file. + +

    RecordWriter implementations write the job outputs to the + {@link FileSystem}. + + @see OutputFormat]]> + + + + + + + + + + + + + + + Reduces values for a given key. + +

    The framework calls this method for each + <key, (list of values)> pair in the grouped inputs. + Output values must be of the same type as input values. Input keys must + not be altered. The framework will reuse the key and value objects + that are passed into the reduce, therefore the application should clone + the objects they want to keep a copy of. In many cases, all values are + combined into zero or one value. +

    + +

    Output pairs are collected with calls to + {@link OutputCollector#collect(Object,Object)}.

    + +

    Applications can use the {@link Reporter} provided to report progress + or just indicate that they are alive. In scenarios where the application + takes a significant amount of time to process individual key/value + pairs, this is crucial since the framework might assume that the task has + timed-out and kill that task. The other way of avoiding this is to set + + mapreduce.task.timeout to a high-enough value (or even zero for no + time-outs).

    + + @param key the key. + @param values the list of values to reduce. + @param output to collect keys and combined values. + @param reporter facility to report progress.]]> +     + + + The number of Reducers for the job is set by the user via + {@link JobConf#setNumReduceTasks(int)}. Reducer implementations + can access the {@link JobConf} for the job via the + {@link JobConfigurable#configure(JobConf)} method and initialize themselves. + Similarly they can use the {@link Closeable#close()} method for + de-initialization.

    + +

    Reducer has 3 primary phases:

    +
      +
    1. + + Shuffle + +

      Reducer is input the grouped output of a {@link Mapper}. + In the phase the framework, for each Reducer, fetches the + relevant partition of the output of all the Mappers, via HTTP. +

      +
      2. + +
    2. + Sort + +

      The framework groups Reducer inputs by keys + (since different Mappers may have output the same key) in this + stage.

      + +

      The shuffle and sort phases occur simultaneously i.e. while outputs are + being fetched they are merged.

      + + SecondarySort + +

      If equivalence rules for keys while grouping the intermediates are + different from those for grouping keys before reduction, then one may + specify a Comparator via + {@link JobConf#setOutputValueGroupingComparator(Class)}.Since + {@link JobConf#setOutputKeyComparatorClass(Class)} can be used to + control how intermediate keys are grouped, these can be used in conjunction + to simulate secondary sort on values.

      + + + For example, say that you want to find duplicate web pages and tag them + all with the url of the "best" known example. You would set up the job + like: +
        +
      • Map Input Key: url
        • +
      • Map Input Value: document
        • +
      • Map Output Key: document checksum, url pagerank
        • +
      • Map Output Value: url
        • +
      • Partitioner: by checksum
        • +
      • OutputKeyComparator: by checksum and then decreasing pagerank
        • +
      • OutputValueGroupingComparator: by checksum
        • +
      +
      3. + +
    3. + Reduce + +

      In this phase the + {@link #reduce(Object, Iterator, OutputCollector, Reporter)} + method is called for each <key, (list of values)> pair in + the grouped inputs.

      +

      The output of the reduce task is typically written to the + {@link FileSystem} via + {@link OutputCollector#collect(Object, Object)}.

      +
      4. +
    + +

    The output of the Reducer is not re-sorted.

    + +

    Example:

    + 

    +     public class MyReducer<K extends WritableComparable, V extends Writable> 
+     extends MapReduceBase implements Reducer<K, V, K, V> {
+     
+       static enum MyCounters { NUM_RECORDS }
+        
+       private String reduceTaskId;
+       private int noKeys = 0;
+       
+       public void configure(JobConf job) {
+         reduceTaskId = job.get(JobContext.TASK_ATTEMPT_ID);
+       }
+       
+       public void reduce(K key, Iterator<V> values,
+                          OutputCollector<K, V> output, 
+                          Reporter reporter)
+       throws IOException {
+       
+         // Process
+         int noValues = 0;
+         while (values.hasNext()) {
+           V value = values.next();
+           
+           // Increment the no. of values for this key
+           ++noValues;
+           
+           // Process the <key, value> pair (assume this takes a while)
+           // ...
+           // ...
+           
+           // Let the framework know that we are alive, and kicking!
+           if ((noValues%10) == 0) {
+             reporter.progress();
+           }
+         
+           // Process some more
+           // ...
+           // ...
+           
+           // Output the <key, value> 
+           output.collect(key, value);
+         }
+         
+         // Increment the no. of <key, list of values> pairs processed
+         ++noKeys;
+         
+         // Increment counters
+         reporter.incrCounter(NUM_RECORDS, 1);
+         
+         // Every 100 keys update application-level status
+         if ((noKeys%100) == 0) {
+           reporter.setStatus(reduceTaskId + " processed " + noKeys);
+         }
+       }
+     }
+
    + + @see Mapper + @see Partitioner + @see Reporter + @see MapReduceBase]]> +     +     + + + + + + + + + + + + + + Counter of the given group/name.]]> + + + + + + + Counter of the given group/name.]]> + + + + + + + Enum. + @param amount A non-negative amount by which the counter is to + be incremented.]]> + + + + + + + + + + + + + + InputSplit that the map is reading from. + @throws UnsupportedOperationException if called outside a mapper]]> + + + + + + + + + + + + + + {@link Mapper} and {@link Reducer} can use the Reporter + provided to report progress or just indicate that they are alive. In + scenarios where the application takes significant amount of time to + process individual key/value pairs, this is crucial since the framework + might assume that the task has timed-out and kill that task. + +

    Applications can also update {@link Counters} via the provided + Reporter .

    + + @see Progressable + @see Counters]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + progress of the job's map-tasks, as a float between 0.0 + and 1.0. When all map tasks have completed, the function returns 1.0. + + @return the progress of the job's map-tasks. + @throws IOException]]> + + + + + + progress of the job's reduce-tasks, as a float between 0.0 + and 1.0. When all reduce tasks have completed, the function returns 1.0. + + @return the progress of the job's reduce-tasks. + @throws IOException]]> + + + + + + progress of the job's cleanup-tasks, as a float between 0.0 + and 1.0. When all cleanup tasks have completed, the function returns 1.0. + + @return the progress of the job's cleanup-tasks. + @throws IOException]]> + + + + + + progress of the job's setup-tasks, as a float between 0.0 + and 1.0. When all setup tasks have completed, the function returns 1.0. + + @return the progress of the job's setup-tasks. + @throws IOException]]> + + + + + + true if the job is complete, else false. + @throws IOException]]> + + + + + + true if the job succeeded, else false. + @throws IOException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if the job retired, else false. + @throws IOException]]> + + + + + + + + + + RunningJob is the user-interface to query for details on a + running Map-Reduce job. + +

    Clients can get hold of RunningJob via the {@link JobClient} + and then query the running-job for details such as name, configuration, + progress etc.

    + + @see JobClient]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + This allows the user to specify the key class to be different + from the actual class ({@link BytesWritable}) used for writing

    + + @param conf the {@link JobConf} to modify + @param theClass the SequenceFile output key class.]]> +     +     + + + + + This allows the user to specify the value class to be different + from the actual class ({@link BytesWritable}) used for writing

    + + @param conf the {@link JobConf} to modify + @param theClass the SequenceFile output key class.]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if auto increment + {@link SkipBadRecords#COUNTER_MAP_PROCESSED_RECORDS}. + false otherwise.]]> + + + + + + + + + + + + + true if auto increment + {@link SkipBadRecords#COUNTER_REDUCE_PROCESSED_GROUPS}. + false otherwise.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Hadoop provides an optional mode of execution in which the bad records + are detected and skipped in further attempts. + +

    This feature can be used when map/reduce tasks crashes deterministically on + certain input. This happens due to bugs in the map/reduce function. The usual + course would be to fix these bugs. But sometimes this is not possible; + perhaps the bug is in third party libraries for which the source code is + not available. Due to this, the task never reaches to completion even with + multiple attempts and complete data for that task is lost.

    + +

    With this feature, only a small portion of data is lost surrounding + the bad record, which may be acceptable for some user applications. + see {@link SkipBadRecords#setMapperMaxSkipRecords(Configuration, long)}

    + +

    The skipping mode gets kicked off after certain no of failures + see {@link SkipBadRecords#setAttemptsToStartSkipping(Configuration, int)}

    + +

    In the skipping mode, the map/reduce task maintains the record range which + is getting processed at all times. Before giving the input to the + map/reduce function, it sends this record range to the Task tracker. + If task crashes, the Task tracker knows which one was the last reported + range. On further attempts that range get skipped.

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + all task attempt IDs + of any jobtracker, in any job, of the first + map task, we would use : + 
     
+ TaskAttemptID.getTaskAttemptIDsPattern(null, null, true, 1, null);
+
    + which will return : + 
     "attempt_[^_]*_[0-9]*_m_000001_[0-9]*"
    + @param jtIdentifier jobTracker identifier, or null + @param jobId job number, or null + @param isMap whether the tip is a map, or null + @param taskId taskId number, or null + @param attemptId the task attempt number, or null + @return a regex pattern matching TaskAttemptIDs]]> +     +     + + + + + + + + all task attempt IDs + of any jobtracker, in any job, of the first + map task, we would use : + 
     
+ TaskAttemptID.getTaskAttemptIDsPattern(null, null, TaskType.MAP, 1, null);
+
    + which will return : + 
     "attempt_[^_]*_[0-9]*_m_000001_[0-9]*"
    + @param jtIdentifier jobTracker identifier, or null + @param jobId job number, or null + @param type the {@link TaskType} + @param taskId taskId number, or null + @param attemptId the task attempt number, or null + @return a regex pattern matching TaskAttemptIDs]]> +     +     + + + An example TaskAttemptID is : + attempt_200707121733_0003_m_000005_0 , which represents the + zeroth task attempt for the fifth map task in the third job + running at the jobtracker started at 200707121733. +

    + Applications should never construct or parse TaskAttemptID strings + , but rather use appropriate constructors or {@link #forName(String)} + method. + + @see JobID + @see TaskID]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + the first map task + of any jobtracker, of any job, we would use : + 

     
+ TaskID.getTaskIDsPattern(null, null, true, 1);
+
    + which will return : + 
     "task_[^_]*_[0-9]*_m_000001*"
    + @param jtIdentifier jobTracker identifier, or null + @param jobId job number, or null + @param isMap whether the tip is a map, or null + @param taskId taskId number, or null + @return a regex pattern matching TaskIDs + @deprecated Use {@link TaskID#getTaskIDsPattern(String, Integer, TaskType, + Integer)}]]> +     + + + + + + + + the first map task + of any jobtracker, of any job, we would use : + 
     
+ TaskID.getTaskIDsPattern(null, null, true, 1);
+
    + which will return : + 
     "task_[^_]*_[0-9]*_m_000001*"
    + @param jtIdentifier jobTracker identifier, or null + @param jobId job number, or null + @param type the {@link TaskType}, or null + @param taskId taskId number, or null + @return a regex pattern matching TaskIDs]]> +     +     + + + + + + + An example TaskID is : + task_200707121733_0003_m_000005 , which represents the + fifth map task in the third job running at the jobtracker + started at 200707121733. +

    + Applications should never construct or parse TaskID strings + , but rather use appropriate constructors or {@link #forName(String)} + method. + + @see JobID + @see TaskAttemptID]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if the Job was added.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ([,]*) + func ::= tbl(,"") + class ::= @see java.lang.Class#forName(java.lang.String) + path ::= @see org.apache.hadoop.fs.Path#Path(java.lang.String) + } + Reads expression from the mapred.join.expr property and + user-supplied join types from mapred.join.define.<ident> + types. Paths supplied to tbl are given as input paths to the + InputFormat class listed. + @see #compose(java.lang.String, java.lang.Class, java.lang.String...)]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ,

    ) }]]> + + + + + + + + (tbl(,),tbl(,),...,tbl(,)) }]]> + + + + + + + + (tbl(,),tbl(,),...,tbl(,)) }]]> + + + + mapred.join.define.<ident> to a classname. In the expression + mapred.join.expr, the identifier will be assumed to be a + ComposableRecordReader. + mapred.join.keycomparator can be a classname used to compare keys + in the join. + @see #setFormat + @see JoinRecordReader + @see MultiFilterRecordReader]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ...... + }]]> + + + + + + + + + + + + + + + + + + + + + capacity children to position + id in the parent reader. + The id of a root CompositeRecordReader is -1 by convention, but relying + on this is not recommended.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + override(S1,S2,S3) will prefer values + from S3 over S2, and values from S2 over S1 for all keys + emitted from all sources.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + It has to be specified how key and values are passed from one element of + the chain to the next, by value or by reference. If a Mapper leverages the + assumed semantics that the key and values are not modified by the collector + 'by value' must be used. If the Mapper does not expect this semantics, as + an optimization to avoid serialization and deserialization 'by reference' + can be used. +

    + For the added Mapper the configuration given for it, + mapperConf, have precedence over the job's JobConf. This + precedence is in effect when the task is running. +

    + IMPORTANT: There is no need to specify the output key/value classes for the + ChainMapper, this is done by the addMapper for the last mapper in the chain +

    + + @param job job's JobConf to add the Mapper class. + @param klass the Mapper class to add. + @param inputKeyClass mapper input key class. + @param inputValueClass mapper input value class. + @param outputKeyClass mapper output key class. + @param outputValueClass mapper output value class. + @param byValue indicates if key/values should be passed by value + to the next Mapper in the chain, if any. + @param mapperConf a JobConf with the configuration for the Mapper + class. It is recommended to use a JobConf without default values using the + JobConf(boolean loadDefaults) constructor with FALSE.]]> + + + + + + + If this method is overriden super.configure(...) should be + invoked at the beginning of the overwriter method.]]> + + + + + + + + + + map(...) methods of the Mappers in the chain.]]> + + + + + + + If this method is overriden super.close() should be + invoked at the end of the overwriter method.]]> + + + + + The Mapper classes are invoked in a chained (or piped) fashion, the output of + the first becomes the input of the second, and so on until the last Mapper, + the output of the last Mapper will be written to the task's output. +

    + The key functionality of this feature is that the Mappers in the chain do not + need to be aware that they are executed in a chain. This enables having + reusable specialized Mappers that can be combined to perform composite + operations within a single task. +

    + Special care has to be taken when creating chains that the key/values output + by a Mapper are valid for the following Mapper in the chain. It is assumed + all Mappers and the Reduce in the chain use maching output and input key and + value classes as no conversion is done by the chaining code. +

    + Using the ChainMapper and the ChainReducer classes is possible to compose + Map/Reduce jobs that look like [MAP+ / REDUCE MAP*]. And + immediate benefit of this pattern is a dramatic reduction in disk IO. +

    + IMPORTANT: There is no need to specify the output key/value classes for the + ChainMapper, this is done by the addMapper for the last mapper in the chain. +

    + ChainMapper usage pattern: +

    + 

    + ...
+ conf.setJobName("chain");
+ conf.setInputFormat(TextInputFormat.class);
+ conf.setOutputFormat(TextOutputFormat.class);
+
+ JobConf mapAConf = new JobConf(false);
+ ...
+ ChainMapper.addMapper(conf, AMap.class, LongWritable.class, Text.class,
+   Text.class, Text.class, true, mapAConf);
+
+ JobConf mapBConf = new JobConf(false);
+ ...
+ ChainMapper.addMapper(conf, BMap.class, Text.class, Text.class,
+   LongWritable.class, Text.class, false, mapBConf);
+
+ JobConf reduceConf = new JobConf(false);
+ ...
+ ChainReducer.setReducer(conf, XReduce.class, LongWritable.class, Text.class,
+   Text.class, Text.class, true, reduceConf);
+
+ ChainReducer.addMapper(conf, CMap.class, Text.class, Text.class,
+   LongWritable.class, Text.class, false, null);
+
+ ChainReducer.addMapper(conf, DMap.class, LongWritable.class, Text.class,
+   LongWritable.class, LongWritable.class, true, null);
+
+ FileInputFormat.setInputPaths(conf, inDir);
+ FileOutputFormat.setOutputPath(conf, outDir);
+ ...
+
+ JobClient jc = new JobClient(conf);
+ RunningJob job = jc.submitJob(conf);
+ ...
+
    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + It has to be specified how key and values are passed from one element of + the chain to the next, by value or by reference. If a Reducer leverages the + assumed semantics that the key and values are not modified by the collector + 'by value' must be used. If the Reducer does not expect this semantics, as + an optimization to avoid serialization and deserialization 'by reference' + can be used. +

    + For the added Reducer the configuration given for it, + reducerConf, have precedence over the job's JobConf. This + precedence is in effect when the task is running. +

    + IMPORTANT: There is no need to specify the output key/value classes for the + ChainReducer, this is done by the setReducer or the addMapper for the last + element in the chain. + + @param job job's JobConf to add the Reducer class. + @param klass the Reducer class to add. + @param inputKeyClass reducer input key class. + @param inputValueClass reducer input value class. + @param outputKeyClass reducer output key class. + @param outputValueClass reducer output value class. + @param byValue indicates if key/values should be passed by value + to the next Mapper in the chain, if any. + @param reducerConf a JobConf with the configuration for the Reducer + class. It is recommended to use a JobConf without default values using the + JobConf(boolean loadDefaults) constructor with FALSE.]]> + + + + + + + + + + + + + + It has to be specified how key and values are passed from one element of + the chain to the next, by value or by reference. If a Mapper leverages the + assumed semantics that the key and values are not modified by the collector + 'by value' must be used. If the Mapper does not expect this semantics, as + an optimization to avoid serialization and deserialization 'by reference' + can be used. +

    + For the added Mapper the configuration given for it, + mapperConf, have precedence over the job's JobConf. This + precedence is in effect when the task is running. +

    + IMPORTANT: There is no need to specify the output key/value classes for the + ChainMapper, this is done by the addMapper for the last mapper in the chain + . + + @param job chain job's JobConf to add the Mapper class. + @param klass the Mapper class to add. + @param inputKeyClass mapper input key class. + @param inputValueClass mapper input value class. + @param outputKeyClass mapper output key class. + @param outputValueClass mapper output value class. + @param byValue indicates if key/values should be passed by value + to the next Mapper in the chain, if any. + @param mapperConf a JobConf with the configuration for the Mapper + class. It is recommended to use a JobConf without default values using the + JobConf(boolean loadDefaults) constructor with FALSE.]]> + + + + + + + If this method is overriden super.configure(...) should be + invoked at the beginning of the overwriter method.]]> + + + + + + + + + + reduce(...) method of the Reducer with the + map(...) methods of the Mappers in the chain.]]> + + + + + + + If this method is overriden super.close() should be + invoked at the end of the overwriter method.]]> + + + + + For each record output by the Reducer, the Mapper classes are invoked in a + chained (or piped) fashion, the output of the first becomes the input of the + second, and so on until the last Mapper, the output of the last Mapper will + be written to the task's output. +

    + The key functionality of this feature is that the Mappers in the chain do not + need to be aware that they are executed after the Reducer or in a chain. + This enables having reusable specialized Mappers that can be combined to + perform composite operations within a single task. +

    + Special care has to be taken when creating chains that the key/values output + by a Mapper are valid for the following Mapper in the chain. It is assumed + all Mappers and the Reduce in the chain use maching output and input key and + value classes as no conversion is done by the chaining code. +

    + Using the ChainMapper and the ChainReducer classes is possible to compose + Map/Reduce jobs that look like [MAP+ / REDUCE MAP*]. And + immediate benefit of this pattern is a dramatic reduction in disk IO. +

    + IMPORTANT: There is no need to specify the output key/value classes for the + ChainReducer, this is done by the setReducer or the addMapper for the last + element in the chain. +

    + ChainReducer usage pattern: +

    + 

    + ...
+ conf.setJobName("chain");
+ conf.setInputFormat(TextInputFormat.class);
+ conf.setOutputFormat(TextOutputFormat.class);
+
+ JobConf mapAConf = new JobConf(false);
+ ...
+ ChainMapper.addMapper(conf, AMap.class, LongWritable.class, Text.class,
+   Text.class, Text.class, true, mapAConf);
+
+ JobConf mapBConf = new JobConf(false);
+ ...
+ ChainMapper.addMapper(conf, BMap.class, Text.class, Text.class,
+   LongWritable.class, Text.class, false, mapBConf);
+
+ JobConf reduceConf = new JobConf(false);
+ ...
+ ChainReducer.setReducer(conf, XReduce.class, LongWritable.class, Text.class,
+   Text.class, Text.class, true, reduceConf);
+
+ ChainReducer.addMapper(conf, CMap.class, Text.class, Text.class,
+   LongWritable.class, Text.class, false, null);
+
+ ChainReducer.addMapper(conf, DMap.class, LongWritable.class, Text.class,
+   LongWritable.class, LongWritable.class, true, null);
+
+ FileInputFormat.setInputPaths(conf, inDir);
+ FileOutputFormat.setOutputPath(conf, outDir);
+ ...
+
+ JobClient jc = new JobClient(conf);
+ RunningJob job = jc.submitJob(conf);
+ ...
+
    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + RecordReader's for CombineFileSplit's. + @see CombineFileSplit]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + CombineFileRecordReader. + + Subclassing is needed to get a concrete record reader wrapper because of the + constructor requirement. + + @see CombineFileRecordReader + @see CombineFileInputFormat]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + CombineFileInputFormat-equivalent for + SequenceFileInputFormat. + + @see CombineFileInputFormat]]> + + + + + + + + + + + + + + + CombineFileInputFormat-equivalent for + TextInputFormat. + + @see CombineFileInputFormat]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if the name output is multi, false + if it is single. If the name output is not defined it returns + false]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + By default these counters are disabled. +

    + MultipleOutputs supports counters, by default the are disabled. + The counters group is the {@link MultipleOutputs} class name. +

    + The names of the counters are the same as the named outputs. For multi + named outputs the name of the counter is the concatenation of the named + output, and underscore '_' and the multiname. + + @param conf job conf to enableadd the named output. + @param enabled indicates if the counters will be enabled or not.]]> +     +     + + + + + By default these counters are disabled. +

    + MultipleOutputs supports counters, by default the are disabled. + The counters group is the {@link MultipleOutputs} class name. +

    + The names of the counters are the same as the named outputs. For multi + named outputs the name of the counter is the concatenation of the named + output, and underscore '_' and the multiname. + + + @param conf job conf to enableadd the named output. + @return TRUE if the counters are enabled, FALSE if they are disabled.]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + If overriden subclasses must invoke super.close() at the + end of their close() + + @throws java.io.IOException thrown if any of the MultipleOutput files + could not be closed properly.]]> + + + + OutputCollector passed to + the map() and reduce() methods of the + Mapper and Reducer implementations. +

    + Each additional output, or named output, may be configured with its own + OutputFormat, with its own key class and with its own value + class. +

    + A named output can be a single file or a multi file. The later is refered as + a multi named output. +

    + A multi named output is an unbound set of files all sharing the same + OutputFormat, key class and value class configuration. +

    + When named outputs are used within a Mapper implementation, + key/values written to a name output are not part of the reduce phase, only + key/values written to the job OutputCollector are part of the + reduce phase. +

    + MultipleOutputs supports counters, by default the are disabled. The counters + group is the {@link MultipleOutputs} class name. +

    + The names of the counters are the same as the named outputs. For multi + named outputs the name of the counter is the concatenation of the named + output, and underscore '_' and the multiname. +

    + Job configuration usage pattern is: + 

    +
+ JobConf conf = new JobConf();
+
+ conf.setInputPath(inDir);
+ FileOutputFormat.setOutputPath(conf, outDir);
+
+ conf.setMapperClass(MOMap.class);
+ conf.setReducerClass(MOReduce.class);
+ ...
+
+ // Defines additional single text based output 'text' for the job
+ MultipleOutputs.addNamedOutput(conf, "text", TextOutputFormat.class,
+ LongWritable.class, Text.class);
+
+ // Defines additional multi sequencefile based output 'sequence' for the
+ // job
+ MultipleOutputs.addMultiNamedOutput(conf, "seq",
+   SequenceFileOutputFormat.class,
+   LongWritable.class, Text.class);
+ ...
+
+ JobClient jc = new JobClient();
+ RunningJob job = jc.submitJob(conf);
+
+ ...
+
    +

    + Job configuration usage pattern is: + 

    +
+ public class MOReduce implements
+   Reducer<WritableComparable, Writable> {
+ private MultipleOutputs mos;
+
+ public void configure(JobConf conf) {
+ ...
+ mos = new MultipleOutputs(conf);
+ }
+
+ public void reduce(WritableComparable key, Iterator<Writable> values,
+ OutputCollector output, Reporter reporter)
+ throws IOException {
+ ...
+ mos.getCollector("text", reporter).collect(key, new Text("Hello"));
+ mos.getCollector("seq", "A", reporter).collect(key, new Text("Bye"));
+ mos.getCollector("seq", "B", reporter).collect(key, new Text("Chau"));
+ ...
+ }
+
+ public void close() throws IOException {
+ mos.close();
+ ...
+ }
+
+ }
+
    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + It can be used instead of the default implementation, + of {@link org.apache.hadoop.mapred.MapRunner}, when the Map + operation is not CPU bound in order to improve throughput. +

    + Map implementations using this MapRunnable must be thread-safe. +

    + The Map-Reduce job has to be configured to use this MapRunnable class (using + the JobConf.setMapRunnerClass method) and + the number of threads the thread-pool can use with the + mapred.map.multithreadedrunner.threads property, its default + value is 10 threads. +

    ]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + R reduces, there are R-1 + keys in the SequenceFile. + @deprecated Use + {@link #setPartitionFile(Configuration, Path)} + instead]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Cluster. + @throws IOException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ClusterMetrics provides clients with information such as: +

      +
    1. + Size of the cluster. +
      2. +
    2. + Number of blacklisted and decommissioned trackers. +
      3. +
    3. + Slot capacity of the cluster. +
      4. +
    4. + The number of currently occupied/reserved map and reduce slots. +
      5. +
    5. + The number of currently running map and reduce tasks. +
      6. +
    6. + The number of job submissions. +
      7. +
    + +

    Clients can query for the latest ClusterMetrics, via + {@link Cluster#getClusterStatus()}.

    + + @see Cluster]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Counters represent global counters, defined either by the + Map-Reduce framework or applications. Each Counter is named by + an {@link Enum} and has a long for the value.

    + +

    Counters are bunched into Groups, each comprising of + counters from a particular Enum class.]]> + + + + + + + + + + + + + + + + + + + + + the type of counter + @param the type of counter group + @param counters the old counters object]]> + + + + Counters holds per job/task counters, defined either by the + Map-Reduce framework or applications. Each Counter can be of + any {@link Enum} type.

    + +

    Counters are bunched into {@link CounterGroup}s, each + comprising of counters from a particular Enum class.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Each {@link InputSplit} is then assigned to an individual {@link Mapper} + for processing.

    + +

    Note: The split is a logical split of the inputs and the + input files are not physically split into chunks. For e.g. a split could + be <input-file-path, start, offset> tuple. The InputFormat + also creates the {@link RecordReader} to read the {@link InputSplit}. + + @param context job configuration. + @return an array of {@link InputSplit}s for the job.]]> + + + + + + + + + + + + + InputFormat describes the input-specification for a + Map-Reduce job. + +

    The Map-Reduce framework relies on the InputFormat of the + job to:

    +

      +
    1. + Validate the input-specification of the job. +
    2. + Split-up the input file(s) into logical {@link InputSplit}s, each of + which is then assigned to an individual {@link Mapper}. +
      3. +
    3. + Provide the {@link RecordReader} implementation to be used to glean + input records from the logical InputSplit for processing by + the {@link Mapper}. +
      4. +
    + +

    The default behavior of file-based {@link InputFormat}s, typically + sub-classes of {@link FileInputFormat}, is to split the + input into logical {@link InputSplit}s based on the total size, in + bytes, of the input files. However, the {@link FileSystem} blocksize of + the input files is treated as an upper bound for input splits. A lower bound + on the split size can be set via + + mapreduce.input.fileinputformat.split.minsize.

    + +

    Clearly, logical splits based on input-size is insufficient for many + applications since record boundaries are to respected. In such cases, the + application has to also implement a {@link RecordReader} on whom lies the + responsibility to respect record-boundaries and present a record-oriented + view of the logical InputSplit to the individual task. + + @see InputSplit + @see RecordReader + @see FileInputFormat]]> + + + + + + + + + + + + + + + + + + + + + + + + + SplitLocationInfos describing how the split + data is stored at each location. A null value indicates that all the + locations have the data stored on disk. + @throws IOException]]> + + + + InputSplit represents the data to be processed by an + individual {@link Mapper}. + +

    Typically, it presents a byte-oriented view on the input and is the + responsibility of {@link RecordReader} of the job to process this and present + a record-oriented view. + + @see InputFormat + @see RecordReader]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Job makes a copy of the Configuration so + that any necessary internal modifications do not reflect on the incoming + parameter. + + A Cluster will be created from the conf parameter only when it's needed. + + @param conf the configuration + @return the {@link Job} , with no connection to a cluster yet. + @throws IOException]]> + + + + + + + + Job makes a copy of the Configuration so + that any necessary internal modifications do not reflect on the incoming + parameter. + + @param conf the configuration + @return the {@link Job} , with no connection to a cluster yet. + @throws IOException]]> + + + + + + + + Job makes a copy of the Configuration so + that any necessary internal modifications do not reflect on the incoming + parameter. + + @param status job status + @param conf job configuration + @return the {@link Job} , with no connection to a cluster yet. + @throws IOException]]> + + + + + + + Job makes a copy of the Configuration so + that any necessary internal modifications do not reflect on the incoming + parameter. + + @param ignored + @return the {@link Job} , with no connection to a cluster yet. + @throws IOException + @deprecated Use {@link #getInstance()}]]> + + + + + + + + Job makes a copy of the Configuration so + that any necessary internal modifications do not reflect on the incoming + parameter. + + @param ignored + @param conf job configuration + @return the {@link Job} , with no connection to a cluster yet. + @throws IOException + @deprecated Use {@link #getInstance(Configuration)}]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + progress of the job's map-tasks, as a float between 0.0 + and 1.0. When all map tasks have completed, the function returns 1.0. + + @return the progress of the job's map-tasks. + @throws IOException]]> + + + + + + progress of the job's reduce-tasks, as a float between 0.0 + and 1.0. When all reduce tasks have completed, the function returns 1.0. + + @return the progress of the job's reduce-tasks. + @throws IOException]]> + + + + + + + progress of the job's cleanup-tasks, as a float between 0.0 + and 1.0. When all cleanup tasks have completed, the function returns 1.0. + + @return the progress of the job's cleanup-tasks. + @throws IOException]]> + + + + + + progress of the job's setup-tasks, as a float between 0.0 + and 1.0. When all setup tasks have completed, the function returns 1.0. + + @return the progress of the job's setup-tasks. + @throws IOException]]> + + + + + + true if the job is complete, else false. + @throws IOException]]> + + + + + + true if the job succeeded, else false. + @throws IOException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + InputFormat to use + @throws IllegalStateException if the job is submitted]]> + + + + + + + OutputFormat to use + @throws IllegalStateException if the job is submitted]]> + + + + + + + Mapper to use + @throws IllegalStateException if the job is submitted]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Reducer to use + @throws IllegalStateException if the job is submitted]]> + + + + + + + Partitioner to use + @throws IllegalStateException if the job is submitted]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if speculative execution + should be turned on, else false.]]> + + + + + + true if speculative execution + should be turned on for map tasks, + else false.]]> + + + + + + true if speculative execution + should be turned on for reduce tasks, + else false.]]> + + + + + + true, job-setup and job-cleanup will be + considered from {@link OutputCommitter} + else ignored.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + JobTracker is lost]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + It allows the user to configure the + job, submit it, control its execution, and query the state. The set methods + only work until the job is submitted, afterwards they will throw an + IllegalStateException.

    + +

    + Normally the user creates the application, describes various facets of the + job via {@link Job} and then submits the job and monitor its progress.

    + +

    Here is an example on how to submit a job:

    + 

    +     // Create a new Job
+     Job job = Job.getInstance();
+     job.setJarByClass(MyJob.class);
+     
+     // Specify various job-specific parameters     
+     job.setJobName("myjob");
+     
+     job.setInputPath(new Path("in"));
+     job.setOutputPath(new Path("out"));
+     
+     job.setMapperClass(MyJob.MyMapper.class);
+     job.setReducerClass(MyJob.MyReducer.class);
+
+     // Submit the job, then poll for progress until the job is complete
+     job.waitForCompletion(true);
+
    ]]> +     + + + + + + + + + + + + + + + + + + + + + + + 1. + @return the number of reduce tasks for this job.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + mapred.map.max.attempts + property. If this property is not already set, the default is 4 attempts. + + @return the max number of attempts per map task.]]> + + + + + mapred.reduce.max.attempts + property. If this property is not already set, the default is 4 attempts. + + @return the max number of attempts per reduce task.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + An example JobID is : + job_200707121733_0003 , which represents the third job + running at the jobtracker started at 200707121733. +

    + Applications should never construct or parse JobID strings, but rather + use appropriate constructors or {@link #forName(String)} method. + + @see TaskID + @see TaskAttemptID]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + the key input type to the Mapper + @param the value input type to the Mapper + @param the key output type from the Mapper + @param the value output type from the Mapper]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Maps are the individual tasks which transform input records into a + intermediate records. The transformed intermediate records need not be of + the same type as the input records. A given input pair may map to zero or + many output pairs.

    + +

    The Hadoop Map-Reduce framework spawns one map task for each + {@link InputSplit} generated by the {@link InputFormat} for the job. + Mapper implementations can access the {@link Configuration} for + the job via the {@link JobContext#getConfiguration()}. + +

    The framework first calls + {@link #setup(org.apache.hadoop.mapreduce.Mapper.Context)}, followed by + {@link #map(Object, Object, org.apache.hadoop.mapreduce.Mapper.Context)} + for each key/value pair in the InputSplit. Finally + {@link #cleanup(org.apache.hadoop.mapreduce.Mapper.Context)} is called.

    + +

    All intermediate values associated with a given output key are + subsequently grouped by the framework, and passed to a {@link Reducer} to + determine the final output. Users can control the sorting and grouping by + specifying two key {@link RawComparator} classes.

    + +

    The Mapper outputs are partitioned per + Reducer. Users can control which keys (and hence records) go to + which Reducer by implementing a custom {@link Partitioner}. + +

    Users can optionally specify a combiner, via + {@link Job#setCombinerClass(Class)}, to perform local aggregation of the + intermediate outputs, which helps to cut down the amount of data transferred + from the Mapper to the Reducer. + +

    Applications can specify if and how the intermediate + outputs are to be compressed and which {@link CompressionCodec}s are to be + used via the Configuration.

    + +

    If the job has zero + reduces then the output of the Mapper is directly written + to the {@link OutputFormat} without sorting by keys.

    + +

    Example:

    + 

    + public class TokenCounterMapper 
+     extends Mapper<Object, Text, Text, IntWritable>{
+    
+   private final static IntWritable one = new IntWritable(1);
+   private Text word = new Text();
+   
+   public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
+     StringTokenizer itr = new StringTokenizer(value.toString());
+     while (itr.hasMoreTokens()) {
+       word.set(itr.nextToken());
+       context.write(word, one);
+     }
+   }
+ }
+
    + +

    Applications may override the + {@link #run(org.apache.hadoop.mapreduce.Mapper.Context)} method to exert + greater control on map processing e.g. multi-threaded Mappers + etc.

    + + @see InputFormat + @see JobContext + @see Partitioner + @see Reducer]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + MarkableIterator is a wrapper iterator class that + implements the {@link MarkableIteratorInterface}.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if task output recovery is supported, + false otherwise + @see #recoverTask(TaskAttemptContext) + @deprecated Use {@link #isRecoverySupported(JobContext)} instead.]]> + + + + + + + true repeatable job commit is supported, + false otherwise + @throws IOException]]> + + + + + + + true if task output recovery is supported, + false otherwise + @throws IOException + @see #recoverTask(TaskAttemptContext)]]> + + + + + + + OutputCommitter. This is called from the application master + process, but it is called individually for each task. + + If an exception is thrown the task will be attempted again. + + This may be called multiple times for the same task. But from different + application attempts. + + @param taskContext Context of the task whose output is being recovered + @throws IOException]]> + + + + OutputCommitter describes the commit of task output for a + Map-Reduce job. + +

    The Map-Reduce framework relies on the OutputCommitter of + the job to:

    +

      +
    1. + Setup the job during initialization. For example, create the temporary + output directory for the job during the initialization of the job. +
      2. +
    2. + Cleanup the job after the job completion. For example, remove the + temporary output directory after the job completion. +
      3. +
    3. + Setup the task temporary output. +
      4. +
    4. + Check whether a task needs a commit. This is to avoid the commit + procedure if a task does not need commit. +
      5. +
    5. + Commit of the task output. +
      6. +
    6. + Discard the task commit. +
      7. +
    + The methods in this class can be called from several different processes and + from several different contexts. It is important to know which process and + which context each is called from. Each method should be marked accordingly + in its documentation. It is also important to note that not all methods are + guaranteed to be called once and only once. If a method is not guaranteed to + have this property the output committer needs to handle this appropriately. + Also note it will only be in rare situations where they may be called + multiple times for the same task. + + @see org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter + @see JobContext + @see TaskAttemptContext]]> +     +     + + + + + + + + + + + + + + + + + + + This is to validate the output specification for the job when it is + a job is submitted. Typically checks that it does not already exist, + throwing an exception when it already exists, so that output is not + overwritten.

    + + @param context information about the job + @throws IOException when output should not be attempted]]> +     +     + + + + + + + + + + OutputFormat describes the output-specification for a + Map-Reduce job. + +

    The Map-Reduce framework relies on the OutputFormat of the + job to:

    +

      +
    1. + Validate the output-specification of the job. For e.g. check that the + output directory doesn't already exist. +
    2. + Provide the {@link RecordWriter} implementation to be used to write out + the output files of the job. Output files are stored in a + {@link FileSystem}. +
      3. +
    + + @see RecordWriter]]> +     +     + + + + + + + + + + + Typically a hash function on a all or a subset of the key.

    + + @param key the key to be partioned. + @param value the entry value. + @param numPartitions the total number of partitions. + @return the partition number for the key.]]> +     +     + + Partitioner controls the partitioning of the keys of the + intermediate map-outputs. The key (or a subset of the key) is used to derive + the partition, typically by a hash function. The total number of partitions + is the same as the number of reduce tasks for the job. Hence this controls + which of the m reduce tasks the intermediate key (and hence the + record) is sent for reduction.

    + + Note: If you require your Partitioner class to obtain the Job's configuration + object, implement the {@link Configurable} interface. + + @see Reducer]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + "N/A" + + @return Scheduling information associated to particular Job Queue]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @param ]]> + + + + + + + + + + + + + + + + + + + + + + RecordWriter to future operations. + + @param context the context of the task + @throws IOException]]> + + + + RecordWriter writes the output <key, value> pairs + to an output file. + +

    RecordWriter implementations write the job outputs to the + {@link FileSystem}. + + @see OutputFormat]]> + + + + + + + + + + + + + + + + + + + + + + the class of the input keys + @param the class of the input values + @param the class of the output keys + @param the class of the output values]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Reducer implementations + can access the {@link Configuration} for the job via the + {@link JobContext#getConfiguration()} method.

    + +

    Reducer has 3 primary phases:

    +
      +
    1. + + Shuffle + +

      The Reducer copies the sorted output from each + {@link Mapper} using HTTP across the network.

      +
      2. + +
    2. + Sort + +

      The framework merge sorts Reducer inputs by + keys + (since different Mappers may have output the same key).

      + +

      The shuffle and sort phases occur simultaneously i.e. while outputs are + being fetched they are merged.

      + + SecondarySort + +

      To achieve a secondary sort on the values returned by the value + iterator, the application should extend the key with the secondary + key and define a grouping comparator. The keys will be sorted using the + entire key, but will be grouped using the grouping comparator to decide + which keys and values are sent in the same call to reduce.The grouping + comparator is specified via + {@link Job#setGroupingComparatorClass(Class)}. The sort order is + controlled by + {@link Job#setSortComparatorClass(Class)}.

      + + + For example, say that you want to find duplicate web pages and tag them + all with the url of the "best" known example. You would set up the job + like: +
        +
      • Map Input Key: url
        • +
      • Map Input Value: document
        • +
      • Map Output Key: document checksum, url pagerank
        • +
      • Map Output Value: url
        • +
      • Partitioner: by checksum
        • +
      • OutputKeyComparator: by checksum and then decreasing pagerank
        • +
      • OutputValueGroupingComparator: by checksum
        • +
      +
      3. + +
    3. + Reduce + +

      In this phase the + {@link #reduce(Object, Iterable, org.apache.hadoop.mapreduce.Reducer.Context)} + method is called for each <key, (collection of values)> in + the sorted inputs.

      +

      The output of the reduce task is typically written to a + {@link RecordWriter} via + {@link Context#write(Object, Object)}.

      +
      4. +
    + +

    The output of the Reducer is not re-sorted.

    + +

    Example:

    + 

    + public class IntSumReducer<Key> extends Reducer<Key,IntWritable,
+                                                 Key,IntWritable> {
+   private IntWritable result = new IntWritable();
+ 
+   public void reduce(Key key, Iterable<IntWritable> values,
+                      Context context) throws IOException, InterruptedException {
+     int sum = 0;
+     for (IntWritable val : values) {
+       sum += val.get();
+     }
+     result.set(sum);
+     context.write(key, result);
+   }
+ }
+
    + + @see Mapper + @see Partitioner]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + counterName. + @param counterName counter name + @return the Counter for the given counterName]]> + + + + + + + groupName and + counterName. + @param counterName counter name + @return the Counter for the given groupName and + counterName]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + An example TaskAttemptID is : + attempt_200707121733_0003_m_000005_0 , which represents the + zeroth task attempt for the fifth map task in the third job + running at the jobtracker started at 200707121733. +

    + Applications should never construct or parse TaskAttemptID strings + , but rather use appropriate constructors or {@link #forName(String)} + method. + + @see JobID + @see TaskID]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + An example TaskID is : + task_200707121733_0003_m_000005 , which represents the + fifth map task in the third job running at the jobtracker + started at 200707121733. +

    + Applications should never construct or parse TaskID strings + , but rather use appropriate constructors or {@link #forName(String)} + method. + + @see JobID + @see TaskAttemptID]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + OutputCommitter for the task-attempt]]> + + + + the input key type for the task + @param the input value type for the task + @param the output key type for the task + @param the output value type for the task]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + type of the other counter + @param type of the other counter group + @param counters the counters object to copy + @param groupFactory the factory for new groups]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + type of counter inside the counters + @param type of group inside the counters]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + type of the counter for the group]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The key and values are passed from one element of the chain to the next, by + value. For the added Mapper the configuration given for it, + mapperConf, have precedence over the job's Configuration. This + precedence is in effect when the task is running. +

    +

    + IMPORTANT: There is no need to specify the output key/value classes for the + ChainMapper, this is done by the addMapper for the last mapper in the chain +

    + + @param job + The job. + @param klass + the Mapper class to add. + @param inputKeyClass + mapper input key class. + @param inputValueClass + mapper input value class. + @param outputKeyClass + mapper output key class. + @param outputValueClass + mapper output value class. + @param mapperConf + a configuration for the Mapper class. It is recommended to use a + Configuration without default values using the + Configuration(boolean loadDefaults) constructor with + FALSE.]]> +     + + + + + + + + + + + + The Mapper classes are invoked in a chained (or piped) fashion, the output of + the first becomes the input of the second, and so on until the last Mapper, + the output of the last Mapper will be written to the task's output. +

    +

    + The key functionality of this feature is that the Mappers in the chain do not + need to be aware that they are executed in a chain. This enables having + reusable specialized Mappers that can be combined to perform composite + operations within a single task. +

    +

    + Special care has to be taken when creating chains that the key/values output + by a Mapper are valid for the following Mapper in the chain. It is assumed + all Mappers and the Reduce in the chain use matching output and input key and + value classes as no conversion is done by the chaining code. +

    +

    + Using the ChainMapper and the ChainReducer classes is possible to compose + Map/Reduce jobs that look like [MAP+ / REDUCE MAP*]. And + immediate benefit of this pattern is a dramatic reduction in disk IO. +

    +

    + IMPORTANT: There is no need to specify the output key/value classes for the + ChainMapper, this is done by the addMapper for the last mapper in the chain. +

    + ChainMapper usage pattern: +

    + + 

    + ...
+ Job = new Job(conf);
+
+ Configuration mapAConf = new Configuration(false);
+ ...
+ ChainMapper.addMapper(job, AMap.class, LongWritable.class, Text.class,
+   Text.class, Text.class, true, mapAConf);
+
+ Configuration mapBConf = new Configuration(false);
+ ...
+ ChainMapper.addMapper(job, BMap.class, Text.class, Text.class,
+   LongWritable.class, Text.class, false, mapBConf);
+
+ ...
+
+ job.waitForComplettion(true);
+ ...
+
    ]]> +     +     + + + + + + + + + + + + + + + + The key and values are passed from one element of the chain to the next, by + value. For the added Reducer the configuration given for it, + reducerConf, have precedence over the job's Configuration. + This precedence is in effect when the task is running. +

    +

    + IMPORTANT: There is no need to specify the output key/value classes for the + ChainReducer, this is done by the setReducer or the addMapper for the last + element in the chain. +

    + + @param job + the job + @param klass + the Reducer class to add. + @param inputKeyClass + reducer input key class. + @param inputValueClass + reducer input value class. + @param outputKeyClass + reducer output key class. + @param outputValueClass + reducer output value class. + @param reducerConf + a configuration for the Reducer class. It is recommended to use a + Configuration without default values using the + Configuration(boolean loadDefaults) constructor with + FALSE.]]> +     +     + + + + + + + + + + + + The key and values are passed from one element of the chain to the next, by + value For the added Mapper the configuration given for it, + mapperConf, have precedence over the job's Configuration. This + precedence is in effect when the task is running. +

    +

    + IMPORTANT: There is no need to specify the output key/value classes for the + ChainMapper, this is done by the addMapper for the last mapper in the + chain. +

    + + @param job + The job. + @param klass + the Mapper class to add. + @param inputKeyClass + mapper input key class. + @param inputValueClass + mapper input value class. + @param outputKeyClass + mapper output key class. + @param outputValueClass + mapper output value class. + @param mapperConf + a configuration for the Mapper class. It is recommended to use a + Configuration without default values using the + Configuration(boolean loadDefaults) constructor with + FALSE.]]> +     +     + + + + + + + + + + + For each record output by the Reducer, the Mapper classes are invoked in a + chained (or piped) fashion. The output of the reducer becomes the input of + the first mapper and output of first becomes the input of the second, and so + on until the last Mapper, the output of the last Mapper will be written to + the task's output. +

    +

    + The key functionality of this feature is that the Mappers in the chain do not + need to be aware that they are executed after the Reducer or in a chain. This + enables having reusable specialized Mappers that can be combined to perform + composite operations within a single task. +

    +

    + Special care has to be taken when creating chains that the key/values output + by a Mapper are valid for the following Mapper in the chain. It is assumed + all Mappers and the Reduce in the chain use matching output and input key and + value classes as no conversion is done by the chaining code. +

    +

    Using the ChainMapper and the ChainReducer classes is possible to + compose Map/Reduce jobs that look like [MAP+ / REDUCE MAP*]. And + immediate benefit of this pattern is a dramatic reduction in disk IO.

    +

    + IMPORTANT: There is no need to specify the output key/value classes for the + ChainReducer, this is done by the setReducer or the addMapper for the last + element in the chain. +

    + ChainReducer usage pattern: +

    + + 

    + ...
+ Job = new Job(conf);
+ ....
+
+ Configuration reduceConf = new Configuration(false);
+ ...
+ ChainReducer.setReducer(job, XReduce.class, LongWritable.class, Text.class,
+   Text.class, Text.class, true, reduceConf);
+
+ ChainReducer.addMapper(job, CMap.class, Text.class, Text.class,
+   LongWritable.class, Text.class, false, null);
+
+ ChainReducer.addMapper(job, DMap.class, LongWritable.class, Text.class,
+   LongWritable.class, LongWritable.class, true, null);
+
+ ...
+
+ job.waitForCompletion(true);
+ ...
+
    ]]> +     +     + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + DBInputFormat emits LongWritables containing the record number as + key and DBWritables as value. + + The SQL query, and input class can be using one of the two + setInput methods.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {@link DBOutputFormat} accepts <key,value> pairs, where + key has a type extending DBWritable. Returned {@link RecordWriter} + writes only the key to the database with a batch SQL query.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + DBWritable. DBWritable, is similar to {@link Writable} + except that the {@link #write(PreparedStatement)} method takes a + {@link PreparedStatement}, and {@link #readFields(ResultSet)} + takes a {@link ResultSet}. +

    + Implementations are responsible for writing the fields of the object + to PreparedStatement, and reading the fields of the object from the + ResultSet. + +

    Example:

    + If we have the following table in the database : + 
    + CREATE TABLE MyTable (
+   counter        INTEGER NOT NULL,
+   timestamp      BIGINT  NOT NULL,
+ );
+
    + then we can read/write the tuples from/to the table with : + 

    + public class MyWritable implements Writable, DBWritable {
+   // Some data     
+   private int counter;
+   private long timestamp;
+       
+   //Writable#write() implementation
+   public void write(DataOutput out) throws IOException {
+     out.writeInt(counter);
+     out.writeLong(timestamp);
+   }
+       
+   //Writable#readFields() implementation
+   public void readFields(DataInput in) throws IOException {
+     counter = in.readInt();
+     timestamp = in.readLong();
+   }
+       
+   public void write(PreparedStatement statement) throws SQLException {
+     statement.setInt(1, counter);
+     statement.setLong(2, timestamp);
+   }
+       
+   public void readFields(ResultSet resultSet) throws SQLException {
+     counter = resultSet.getInt(1);
+     timestamp = resultSet.getLong(2);
+   } 
+ }
+
    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + RecordReader's for + CombineFileSplit's. + + @see CombineFileSplit]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + CombineFileRecordReader. + + Subclassing is needed to get a concrete record reader wrapper because of the + constructor requirement. + + @see CombineFileRecordReader + @see CombineFileInputFormat]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + th Path]]> + + + + + + th Path]]> + + + + + + + + + + + th Path]]> + + + + + + + + + + + + + + + + + + + + + + + + + + CombineFileSplit can be used to implement {@link RecordReader}'s, + with reading one record per file. + + @see FileSplit + @see CombineFileInputFormat]]> + + + + + + + + + + + + + + CombineFileInputFormat-equivalent for + SequenceFileInputFormat. + + @see CombineFileInputFormat]]> + + + + + + + + + + + + + + CombineFileInputFormat-equivalent for + TextInputFormat. + + @see CombineFileInputFormat]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + FileInputFormat always returns + true. Implementations that may deal with non-splittable files must + override this method. + + FileInputFormat implementations can override this and return + false to ensure that individual input files are never split-up + so that {@link Mapper}s process entire files. + + @param context the job context + @param filename the file name to check + @return is this file splitable?]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + FileInputFormat is the base class for all file-based + InputFormats. This provides a generic implementation of + {@link #getSplits(JobContext)}. + + Implementations of FileInputFormat can also override the + {@link #isSplitable(JobContext, Path)} method to prevent input files + from being split-up in certain situations. Implementations that may + deal with non-splittable files must override this method, since + the default implementation assumes splitting is always possible.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    or + conf.setInt(FixedLengthInputFormat.FIXED_RECORD_LENGTH, recordLength); +

    + @see FixedLengthRecordReader]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if the Job was added.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ([,]*) + func ::= tbl(,"") + class ::= @see java.lang.Class#forName(java.lang.String) + path ::= @see org.apache.hadoop.fs.Path#Path(java.lang.String) + } + Reads expression from the mapreduce.join.expr property and + user-supplied join types from mapreduce.join.define.<ident> + types. Paths supplied to tbl are given as input paths to the + InputFormat class listed. + @see #compose(java.lang.String, java.lang.Class, java.lang.String...)]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ,

    ) }]]> + + + + + + + + (tbl(,),tbl(,),...,tbl(,)) }]]> + + + + + + + + (tbl(,),tbl(,),...,tbl(,)) }]]> + + + + + + + + mapreduce.join.define.<ident> to a classname. + In the expression mapreduce.join.expr, the identifier will be + assumed to be a ComposableRecordReader. + mapreduce.join.keycomparator can be a classname used to compare + keys in the join. + @see #setFormat + @see JoinRecordReader + @see MultiFilterRecordReader]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ...... + }]]> + + + + + + + + + + + + + + + + + + + + + capacity children to position + id in the parent reader. + The id of a root CompositeRecordReader is -1 by convention, but relying + on this is not recommended.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + override(S1,S2,S3) will prefer values + from S3 over S2, and values from S2 over S1 for all keys + emitted from all sources.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + [<child1>,<child2>,...,<childn>]]]> + + + + + + + out. + TupleWritable format: + {@code + ...... + }]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + the map's input key type + @param the map's input value type + @param the map's output key type + @param the map's output value type + @param job the job + @return the mapper class to run]]> + + + + + + + the map input key type + @param the map input value type + @param the map output key type + @param the map output value type + @param job the job to modify + @param cls the class to use as the mapper]]> + + + + + + + + + + + + + + + + + It can be used instead of the default implementation, + {@link org.apache.hadoop.mapred.MapRunner}, when the Map operation is not CPU + bound in order to improve throughput. +

    + Mapper implementations using this MapRunnable must be thread-safe. +

    + The Map-Reduce job has to be configured with the mapper to use via + {@link #setMapperClass(Job, Class)} and + the number of thread the thread-pool can use with the + {@link #getNumberOfThreads(JobContext)} method. The default + value is 10 threads. +

    ]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + MapContext to be wrapped + @return a wrapped Mapper.Context for custom implementations]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if the job output should be compressed, + false otherwise]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Tasks' Side-Effect Files + +

    Some applications need to create/write-to side-files, which differ from + the actual job-outputs. + +

    In such cases there could be issues with 2 instances of the same TIP + (running simultaneously e.g. speculative tasks) trying to open/write-to the + same file (path) on HDFS. Hence the application-writer will have to pick + unique names per task-attempt (e.g. using the attemptid, say + attempt_200709221812_0001_m_000000_0), not just per TIP.

    + +

    To get around this the Map-Reduce framework helps the application-writer + out by maintaining a special + ${mapreduce.output.fileoutputformat.outputdir}/_temporary/_${taskid} + sub-directory for each task-attempt on HDFS where the output of the + task-attempt goes. On successful completion of the task-attempt the files + in the ${mapreduce.output.fileoutputformat.outputdir}/_temporary/_${taskid} (only) + are promoted to ${mapreduce.output.fileoutputformat.outputdir}. Of course, the + framework discards the sub-directory of unsuccessful task-attempts. This + is completely transparent to the application.

    + +

    The application-writer can take advantage of this by creating any + side-files required in a work directory during execution + of his task i.e. via + {@link #getWorkOutputPath(TaskInputOutputContext)}, and + the framework will move them out similarly - thus she doesn't have to pick + unique paths per task-attempt.

    + +

    The entire discussion holds true for maps of jobs with + reducer=NONE (i.e. 0 reduces) since output of the map, in that case, + goes directly to HDFS.

    + + @return the {@link Path} to the task's temporary output directory + for the map-reduce job.]]> +     +     + + + + + + + + The path can be used to create custom files from within the map and + reduce tasks. The path name will be unique for each task. The path parent + will be the job output directory.

    ls + +

    This method uses the {@link #getUniqueFile} method to make the file name + unique for the task.

    + + @param context the context for the task. + @param name the name for the file. + @param extension the extension for the file + @return a unique path accross all tasks of the job.]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Warning: when the baseOutputPath is a path that resolves + outside of the final job output directory, the directory is created + immediately and then persists through subsequent task retries, breaking + the concept of output committing.]]> + + + + + + + + + + Warning: when the baseOutputPath is a path that resolves + outside of the final job output directory, the directory is created + immediately and then persists through subsequent task retries, breaking + the concept of output committing.]]> + + + + + + + super.close() at the + end of their close()]]> + + + + + Case one: writing to additional outputs other than the job default output. + + Each additional output, or named output, may be configured with its own + OutputFormat, with its own key class and with its own value + class. +

    + +

    + Case two: to write data to different files provided by user +

    + +

    + MultipleOutputs supports counters, by default they are disabled. The + counters group is the {@link MultipleOutputs} class name. The names of the + counters are the same as the output name. These count the number records + written to each output name. +

    + + Usage pattern for job submission: + 
    +
+ Job job = new Job();
+
+ FileInputFormat.setInputPath(job, inDir);
+ FileOutputFormat.setOutputPath(job, outDir);
+
+ job.setMapperClass(MOMap.class);
+ job.setReducerClass(MOReduce.class);
+ ...
+
+ // Defines additional single text based output 'text' for the job
+ MultipleOutputs.addNamedOutput(job, "text", TextOutputFormat.class,
+ LongWritable.class, Text.class);
+
+ // Defines additional sequence-file based output 'sequence' for the job
+ MultipleOutputs.addNamedOutput(job, "seq",
+   SequenceFileOutputFormat.class,
+   LongWritable.class, Text.class);
+ ...
+
+ job.waitForCompletion(true);
+ ...
+
    +

    + Usage in Reducer: + 

    + <K, V> String generateFileName(K k, V v) {
+   return k.toString() + "_" + v.toString();
+ }
+ 
+ public class MOReduce extends
+   Reducer<WritableComparable, Writable,WritableComparable, Writable> {
+ private MultipleOutputs mos;
+ public void setup(Context context) {
+ ...
+ mos = new MultipleOutputs(context);
+ }
+
+ public void reduce(WritableComparable key, Iterator<Writable> values,
+ Context context)
+ throws IOException {
+ ...
+ mos.write("text", , key, new Text("Hello"));
+ mos.write("seq", LongWritable(1), new Text("Bye"), "seq_a");
+ mos.write("seq", LongWritable(2), key, new Text("Chau"), "seq_b");
+ mos.write(key, new Text("value"), generateFileName(key, new Text("value")));
+ ...
+ }
+
+ public void cleanup(Context) throws IOException {
+ mos.close();
+ ...
+ }
+
+ }
+
    + +

    + When used in conjuction with org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat, + MultipleOutputs can mimic the behaviour of MultipleTextOutputFormat and MultipleSequenceFileOutputFormat + from the old Hadoop API - ie, output can be written from the Reducer to more than one location. +

    + +

    + Use MultipleOutputs.write(KEYOUT key, VALUEOUT value, String baseOutputPath) to write key and + value to a path specified by baseOutputPath, with no need to specify a named output. + Warning: when the baseOutputPath passed to MultipleOutputs.write + is a path that resolves outside of the final job output directory, the + directory is created immediately and then persists through subsequent + task retries, breaking the concept of output committing: +

    + + 
    + private MultipleOutputs<Text, Text> out;
+ 
+ public void setup(Context context) {
+   out = new MultipleOutputs<Text, Text>(context);
+   ...
+ }
+ 
+ public void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
+ for (Text t : values) {
+   out.write(key, t, generateFileName(<parameter list...>));
+   }
+ }
+ 
+ protected void cleanup(Context context) throws IOException, InterruptedException {
+   out.close();
+ }
+
    + +

    + Use your own code in generateFileName() to create a custom path to your results. + '/' characters in baseOutputPath will be translated into directory levels in your file system. + Also, append your custom-generated path with "part" or similar, otherwise your output will be -00000, -00001 etc. + No call to context.write() is necessary. See example generateFileName() code below. +

    + + 
    + private String generateFileName(Text k) {
+   // expect Text k in format "Surname|Forename"
+   String[] kStr = k.toString().split("\\|");
+   
+   String sName = kStr[0];
+   String fName = kStr[1];
+
+   // example for k = Smith|John
+   // output written to /user/hadoop/path/to/output/Smith/John-r-00000 (etc)
+   return sName + "/" + fName;
+ }
+
    + +

    + Using MultipleOutputs in this way will still create zero-sized default output, eg part-00000. + To prevent this use LazyOutputFormat.setOutputFormatClass(job, TextOutputFormat.class); + instead of job.setOutputFormatClass(TextOutputFormat.class); in your Hadoop job configuration. +

    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This allows the user to specify the key class to be different + from the actual class ({@link BytesWritable}) used for writing

    + + @param job the {@link Job} to modify + @param theClass the SequenceFile output key class.]]> +     +     + + + + + This allows the user to specify the value class to be different + from the actual class ({@link BytesWritable}) used for writing

    + + @param job the {@link Job} to modify + @param theClass the SequenceFile output key class.]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + bytes[left:(right+1)] in Python syntax. + + @param conf configuration object + @param left left Python-style offset + @param right right Python-style offset]]> + + + + + + + bytes[offset:] in Python syntax. + + @param conf configuration object + @param offset left Python-style offset]]> + + + + + + + bytes[:(offset+1)] in Python syntax. + + @param conf configuration object + @param offset right Python-style offset]]> + + + + + + + + + + + + + + + + + + + + + Partition {@link BinaryComparable} keys using a configurable part of + the bytes array returned by {@link BinaryComparable#getBytes()}.

    + +

    The subarray to be used for the partitioning can be defined by means + of the following properties: +

      +
    • + mapreduce.partition.binarypartitioner.left.offset: + left offset in array (0 by default) +
      • +
    • + mapreduce.partition.binarypartitioner.right.offset: + right offset in array (-1 by default) +
      • +
    + Like in Python, both negative and positive offsets are allowed, but + the meaning is slightly different. In case of an array of length 5, + for instance, the possible offsets are: + 
    
+  +---+---+---+---+---+
+  | B | B | B | B | B |
+  +---+---+---+---+---+
+    0   1   2   3   4
+   -5  -4  -3  -2  -1
+
    + The first row of numbers gives the position of the offsets 0...5 in + the array; the second row gives the corresponding negative offsets. + Contrary to Python, the specified subarray has byte i + and j as first and last element, repectively, when + i and j are the left and right offset. + +

    For Hadoop programs written in Java, it is advisable to use one of + the following static convenience methods for setting the offsets: +

      +
    • {@link #setOffsets}
      • +
    • {@link #setLeftOffset}
      • +
    • {@link #setRightOffset}
      • +
    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + total.order.partitioner.natural.order is not false, a trie + of the first total.order.partitioner.max.trie.depth(2) + 1 bytes + will be built. Otherwise, keys will be located using a binary search of + the partition keyset using the {@link org.apache.hadoop.io.RawComparator} + defined for this job. The input file must be sorted with the same + comparator and contain {@link Job#getNumReduceTasks()} - 1 keys.]]> + + + + + + + + + + + + + + R reduces, there are R-1 + keys in the SequenceFile.]]> + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ReduceContext to be wrapped + @return a wrapped Reducer.Context for custom implementations]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_JobClient_2.8.0.xml b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_JobClient_2.8.0.xml new file mode 100644 index 00000000000..82a6ef18531 --- /dev/null +++ b/hadoop-mapreduce-project/dev-support/jdiff/Apache_Hadoop_MapReduce_JobClient_2.8.0.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Client_2.8.0.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Client_2.8.0.xml new file mode 100644 index 00000000000..152601a536b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Client_2.8.0.xml @@ -0,0 +1,2316 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + In secure mode, YARN verifies access to the application, queue + etc. before accepting the request. +

    + If the user does not have VIEW_APP access then the following + fields in the report will be set to stubbed values: +

      +
    • host - set to "N/A"
      • +
    • RPC port - set to -1
      • +
    • client token - set to "N/A"
      • +
    • diagnostics - set to "N/A"
      • +
    • tracking URL - set to "N/A"
      • +
    • original tracking URL - set to "N/A"
      • +
    • resource usage report - all values are -1
      • +
    + + @param appId + {@link ApplicationId} of the application that needs a report + @return application report + @throws YarnException + @throws IOException]]> +     +     + + + + + + Get a report (ApplicationReport) of all Applications in the cluster. +

    + +

    + If the user does not have VIEW_APP access for an application + then the corresponding report will be filtered as described in + {@link #getApplicationReport(ApplicationId)}. +

    + + @return a list of reports for all applications + @throws YarnException + @throws IOException]]> +     +     + + + + + + + Get a report of the given ApplicationAttempt. +

    + +

    + In secure mode, YARN verifies access to the application, queue + etc. before accepting the request. +

    + + @param applicationAttemptId + {@link ApplicationAttemptId} of the application attempt that needs + a report + @return application attempt report + @throws YarnException + @throws ApplicationAttemptNotFoundException if application attempt + not found + @throws IOException]]> +     +     + + + + + + + Get a report of all (ApplicationAttempts) of Application in the cluster. +

    + + @param applicationId + @return a list of reports for all application attempts for specified + application + @throws YarnException + @throws IOException]]> +     +     + + + + + + + Get a report of the given Container. +

    + +

    + In secure mode, YARN verifies access to the application, queue + etc. before accepting the request. +

    + + @param containerId + {@link ContainerId} of the container that needs a report + @return container report + @throws YarnException + @throws ContainerNotFoundException if container not found + @throws IOException]]> +     +     + + + + + + + Get a report of all (Containers) of ApplicationAttempt in the cluster. +

    + + @param applicationAttemptId + @return a list of reports of all containers for specified application + attempt + @throws YarnException + @throws IOException]]> +     +     +     + + + + + + + + + {@code + AMRMClient.createAMRMClientContainerRequest() + } + @return the newly create AMRMClient instance.]]> + + + + + + + + + + RegisterApplicationMasterResponse + @throws YarnException + @throws IOException]]> + + + + + + + + addContainerRequest are sent to the + ResourceManager. New containers assigned to the master are + retrieved. Status of completed containers and node health updates are also + retrieved. This also doubles up as a heartbeat to the ResourceManager and + must be made periodically. The call may not always return any new + allocations of containers. App should not make concurrent allocate + requests. May cause request loss. + +

    + Note : If the user has not removed container requests that have already + been satisfied, then the re-register may end up sending the entire + container requests to the RM (including matched requests). Which would mean + the RM could end up giving it a lot of new allocated containers. +

    + + @param progressIndicator Indicates progress made by the master + @return the response of the allocate request + @throws YarnException + @throws IOException]]> +     +     + + + + + + + + + + + + + + allocate + @param req Resource request]]> + + + + + + + + + + + + + allocate. + Any previous pending resource change request of the same container will be + removed. + + Application that calls this method is expected to maintain the + Containers that are returned from previous successful + allocations or resource changes. By passing in the existing container and a + target resource capability to this method, the application requests the + ResourceManager to change the existing resource allocation to the target + resource allocation. + + @param container The container returned from the last successful resource + allocation or resource change + @param capability The target resource capability of the container]]> + + + + + + + + + + + + + + + + + + + + + + + + ContainerRequests matching the given + parameters. These ContainerRequests should have been added via + addContainerRequest earlier in the lifecycle. For performance, + the AMRMClient may return its internal collection directly without creating + a copy. Users should not perform mutable operations on the return value. + Each collection in the list contains requests with identical + Resource size that fit in the given capability. In a + collection, requests will be returned in the same order as they were added. + @return Collection of request matching the parameters]]> + + + + + + + + + + + + + AMRMClient. This cache must + be shared with the {@link NMClient} used to manage containers for the + AMRMClient +

    + If a NM token cache is not set, the {@link NMTokenCache#getSingleton()} + singleton instance will be used. + + @param nmTokenCache the NM token cache to use.]]> + + + + + AMRMClient. This cache must be + shared with the {@link NMClient} used to manage containers for the + AMRMClient. +

    + If a NM token cache is not set, the {@link NMTokenCache#getSingleton()} + singleton instance will be used. + + @return the NM token cache.]]> + + + + + + + check to return true for each 1000 ms. + See also {@link #waitFor(com.google.common.base.Supplier, int)} + and {@link #waitFor(com.google.common.base.Supplier, int, int)} + @param check]]> + + + + + + + + check to return true for each + checkEveryMillis ms. + See also {@link #waitFor(com.google.common.base.Supplier, int, int)} + @param check user defined checker + @param checkEveryMillis interval to call check]]> + + + + + + + + + check to return true for each + checkEveryMillis ms. In the main loop, this method will log + the message "waiting in main loop" for each logInterval times + iteration to confirm the thread is alive. + @param check user defined checker + @param checkEveryMillis interval to call check + @param logInterval interval to log for each]]> + + + + + + + + + + + + + + + + + + + + + + + + + + Start an allocated container.

    + +

    The ApplicationMaster or other applications that use the + client must provide the details of the allocated container, including the + Id, the assigned node's Id and the token via {@link Container}. In + addition, the AM needs to provide the {@link ContainerLaunchContext} as + well.

    + + @param container the allocated container + @param containerLaunchContext the context information needed by the + NodeManager to launch the + container + @return a map between the auxiliary service names and their outputs + @throws YarnException + @throws IOException]]> +     +     + + + + + + Increase the resource of a container.

    + +

    The ApplicationMaster or other applications that use the + client must provide the details of the container, including the Id and + the target resource encapsulated in the updated container token via + {@link Container}. +

    + + @param container the container with updated token + @throws YarnException + @throws IOException]]> +     +     + + + + + + + Stop an started container.

    + + @param containerId the Id of the started container + @param nodeId the Id of the NodeManager + + @throws YarnException + @throws IOException]]> +     +     + + + + + + + Query the status of a container.

    + + @param containerId the Id of the started container + @param nodeId the Id of the NodeManager + + @return the status of a container + @throws YarnException + @throws IOException]]> +     +     + + + + Set whether the containers that are started by this client, and are + still running should be stopped when the client stops. By default, the + feature should be enabled.

    However, containers will be stopped only + when service is stopped. i.e. after {@link NMClient#stop()}. + + @param enabled whether the feature is enabled or not]]> +     +     + + + + NMClient. This cache must be + shared with the {@link AMRMClient} that requested the containers managed + by this NMClient +

    + If a NM token cache is not set, the {@link NMTokenCache#getSingleton()} + singleton instance will be used. + + @param nmTokenCache the NM token cache to use.]]> + + + + + NMClient. This cache must be + shared with the {@link AMRMClient} that requested the containers managed + by this NMClient +

    + If a NM token cache is not set, the {@link NMTokenCache#getSingleton()} + singleton instance will be used. + + @return the NM token cache]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + By default Yarn client libraries {@link AMRMClient} and {@link NMClient} use + {@link #getSingleton()} instance of the cache. +

      +
    • + Using the singleton instance of the cache is appropriate when running a + single ApplicationMaster in the same JVM. +
      • +
    • + When using the singleton, users don't need to do anything special, + {@link AMRMClient} and {@link NMClient} are already set up to use the + default singleton {@link NMTokenCache} +
      • +
    + If running multiple Application Masters in the same JVM, a different cache + instance should be used for each Application Master. +
      +
    • + If using the {@link AMRMClient} and the {@link NMClient}, setting up + and using an instance cache is as follows: + 
      +   NMTokenCache nmTokenCache = new NMTokenCache();
+   AMRMClient rmClient = AMRMClient.createAMRMClient();
+   NMClient nmClient = NMClient.createNMClient();
+   nmClient.setNMTokenCache(nmTokenCache);
+   ...
+
      +
      • +
    • + If using the {@link AMRMClientAsync} and the {@link NMClientAsync}, + setting up and using an instance cache is as follows: + 
      +   NMTokenCache nmTokenCache = new NMTokenCache();
+   AMRMClient rmClient = AMRMClient.createAMRMClient();
+   NMClient nmClient = NMClient.createNMClient();
+   nmClient.setNMTokenCache(nmTokenCache);
+   AMRMClientAsync rmClientAsync = new AMRMClientAsync(rmClient, 1000, [AMRM_CALLBACK]);
+   NMClientAsync nmClientAsync = new NMClientAsync("nmClient", nmClient, [NM_CALLBACK]);
+   ...
+
      +
      • +
    • + If using {@link ApplicationMasterProtocol} and + {@link ContainerManagementProtocol} directly, setting up and using an + instance cache is as follows: + 
      +   NMTokenCache nmTokenCache = new NMTokenCache();
+   ...
+   ApplicationMasterProtocol amPro = ClientRMProxy.createRMProxy(conf, ApplicationMasterProtocol.class);
+   ...
+   AllocateRequest allocateRequest = ...
+   ...
+   AllocateResponse allocateResponse = rmClient.allocate(allocateRequest);
+   for (NMToken token : allocateResponse.getNMTokens()) {
+     nmTokenCache.setToken(token.getNodeId().toString(), token.getToken());
+   }
+   ...
+   ContainerManagementProtocolProxy nmPro = ContainerManagementProtocolProxy(conf, nmTokenCache);
+   ...
+   nmPro.startContainer(container, containerContext);
+   ...
+
      +
      • +
    + It is also possible to mix the usage of a client ({@code AMRMClient} or + {@code NMClient}, or the async versions of them) with a protocol proxy + ({@code ContainerManagementProtocolProxy} or + {@code ApplicationMasterProtocol}).]]> +     +     + + + + + + + + + + + + + + The method to claim a resource with the SharedCacheManager. + The client uses a checksum to identify the resource and an + {@link ApplicationId} to identify which application will be using the + resource. +

    + +

    + The SharedCacheManager responds with whether or not the + resource exists in the cache. If the resource exists, a Path + to the resource in the shared cache is returned. If the resource does not + exist, null is returned instead. +

    + + @param applicationId ApplicationId of the application using the resource + @param resourceKey the key (i.e. checksum) that identifies the resource + @return Path to the resource, or null if it does not exist]]> +     +     + + + + + + + The method to release a resource with the SharedCacheManager. + This method is called once an application is no longer using a claimed + resource in the shared cache. The client uses a checksum to identify the + resource and an {@link ApplicationId} to identify which application is + releasing the resource. +

    + +

    + Note: This method is an optimization and the client is not required to call + it for correctness. +

    + + @param applicationId ApplicationId of the application releasing the + resource + @param resourceKey the key (i.e. checksum) that identifies the resource]]> +     +     + + + + + + + + + + +     + + + + + + + + + + + + + + + + Obtain a {@link YarnClientApplication} for a new application, + which in turn contains the {@link ApplicationSubmissionContext} and + {@link org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse} + objects. +

    + + @return {@link YarnClientApplication} built for a new application + @throws YarnException + @throws IOException]]> +     +     + + + + + + + Submit a new application to YARN. It is a blocking call - it + will not return {@link ApplicationId} until the submitted application is + submitted successfully and accepted by the ResourceManager. +

    + +

    + Users should provide an {@link ApplicationId} as part of the parameter + {@link ApplicationSubmissionContext} when submitting a new application, + otherwise it will throw the {@link ApplicationIdNotProvidedException}. +

    + +

    This internally calls {@link ApplicationClientProtocol#submitApplication + (SubmitApplicationRequest)}, and after that, it internally invokes + {@link ApplicationClientProtocol#getApplicationReport + (GetApplicationReportRequest)} and waits till it can make sure that the + application gets properly submitted. If RM fails over or RM restart + happens before ResourceManager saves the application's state, + {@link ApplicationClientProtocol + #getApplicationReport(GetApplicationReportRequest)} will throw + the {@link ApplicationNotFoundException}. This API automatically resubmits + the application with the same {@link ApplicationSubmissionContext} when it + catches the {@link ApplicationNotFoundException}

    + + @param appContext + {@link ApplicationSubmissionContext} containing all the details + needed to submit a new application + @return {@link ApplicationId} of the accepted application + @throws YarnException + @throws IOException + @see #createApplication()]]> +     +     + + + + + + + Fail an application attempt identified by given ID. +

    + + @param applicationAttemptId + {@link ApplicationAttemptId} of the attempt to fail. + @throws YarnException + in case of errors or if YARN rejects the request due to + access-control restrictions. + @throws IOException + @see #getQueueAclsInfo()]]> +     +     + + + + + + + Kill an application identified by given ID. +

    + + @param applicationId + {@link ApplicationId} of the application that needs to be killed + @throws YarnException + in case of errors or if YARN rejects the request due to + access-control restrictions. + @throws IOException + @see #getQueueAclsInfo()]]> +     +     + + + + + + + + Kill an application identified by given ID. +

    + @param applicationId {@link ApplicationId} of the application that needs to + be killed + @param diagnostics for killing an application. + @throws YarnException in case of errors or if YARN rejects the request due + to access-control restrictions. + @throws IOException]]> +     +     + + + + + + + Get a report of the given Application. +

    + +

    + In secure mode, YARN verifies access to the application, queue + etc. before accepting the request. +

    + +

    + If the user does not have VIEW_APP access then the following + fields in the report will be set to stubbed values: +

      +
    • host - set to "N/A"
      • +
    • RPC port - set to -1
      • +
    • client token - set to "N/A"
      • +
    • diagnostics - set to "N/A"
      • +
    • tracking URL - set to "N/A"
      • +
    • original tracking URL - set to "N/A"
      • +
    • resource usage report - all values are -1
      • +
    + + @param appId + {@link ApplicationId} of the application that needs a report + @return application report + @throws YarnException + @throws IOException]]> +     +     + + + + + + + The AMRM token is required for AM to RM scheduling operations. For + managed Application Masters Yarn takes care of injecting it. For unmanaged + Applications Masters, the token must be obtained via this method and set + in the {@link org.apache.hadoop.security.UserGroupInformation} of the + current user. +

    + The AMRM token will be returned only if all the following conditions are + met: +

      +
    • the requester is the owner of the ApplicationMaster
      • +
    • the application master is an unmanaged ApplicationMaster
      • +
    • the application master is in ACCEPTED state
      • +
    + Else this method returns NULL. + + @param appId {@link ApplicationId} of the application to get the AMRM token + @return the AMRM token if available + @throws YarnException + @throws IOException]]> +     +     + + + + + + Get a report (ApplicationReport) of all Applications in the cluster. +

    + +

    + If the user does not have VIEW_APP access for an application + then the corresponding report will be filtered as described in + {@link #getApplicationReport(ApplicationId)}. +

    + + @return a list of reports of all running applications + @throws YarnException + @throws IOException]]> +     +     + + + + + + + Get a report (ApplicationReport) of Applications + matching the given application types in the cluster. +

    + +

    + If the user does not have VIEW_APP access for an application + then the corresponding report will be filtered as described in + {@link #getApplicationReport(ApplicationId)}. +

    + + @param applicationTypes set of application types you are interested in + @return a list of reports of applications + @throws YarnException + @throws IOException]]> +     +     + + + + + + + Get a report (ApplicationReport) of Applications matching the given + application states in the cluster. +

    + +

    + If the user does not have VIEW_APP access for an application + then the corresponding report will be filtered as described in + {@link #getApplicationReport(ApplicationId)}. +

    + + @param applicationStates set of application states you are interested in + @return a list of reports of applications + @throws YarnException + @throws IOException]]> +     +     + + + + + + + + Get a report (ApplicationReport) of Applications matching the given + application types and application states in the cluster. +

    + +

    + If the user does not have VIEW_APP access for an application + then the corresponding report will be filtered as described in + {@link #getApplicationReport(ApplicationId)}. +

    + + @param applicationTypes set of application types you are interested in + @param applicationStates set of application states you are interested in + @return a list of reports of applications + @throws YarnException + @throws IOException]]> +     +     + + + + + + + + + + Get a report (ApplicationReport) of Applications matching the given users, + queues, application types and application states in the cluster. If any of + the params is set to null, it is not used when filtering. +

    + +

    + If the user does not have VIEW_APP access for an application + then the corresponding report will be filtered as described in + {@link #getApplicationReport(ApplicationId)}. +

    + + @param queues set of queues you are interested in + @param users set of users you are interested in + @param applicationTypes set of application types you are interested in + @param applicationStates set of application states you are interested in + @return a list of reports of applications + @throws YarnException + @throws IOException]]> +     +     + + + + + + Get metrics ({@link YarnClusterMetrics}) about the cluster. +

    + + @return cluster metrics + @throws YarnException + @throws IOException]]> +     +     + + + + + + + Get a report of nodes ({@link NodeReport}) in the cluster. +

    + + @param states The {@link NodeState}s to filter on. If no filter states are + given, nodes in all states will be returned. + @return A list of node reports + @throws YarnException + @throws IOException]]> +     +     + + + + + + + Get a delegation token so as to be able to talk to YARN using those tokens. + + @param renewer + Address of the renewer who can renew these tokens when needed by + securely talking to YARN. + @return a delegation token ({@link Token}) that can be used to + talk to YARN + @throws YarnException + @throws IOException]]> + + + + + + + + + Get information ({@link QueueInfo}) about a given queue. +

    + + @param queueName + Name of the queue whose information is needed + @return queue information + @throws YarnException + in case of errors or if YARN rejects the request due to + access-control restrictions. + @throws IOException]]> +     +     + + + + + + Get information ({@link QueueInfo}) about all queues, recursively if there + is a hierarchy +

    + + @return a list of queue-information for all queues + @throws YarnException + @throws IOException]]> +     +     + + + + + + Get information ({@link QueueInfo}) about top level queues. +

    + + @return a list of queue-information for all the top-level queues + @throws YarnException + @throws IOException]]> +     +     + + + + + + + Get information ({@link QueueInfo}) about all the immediate children queues + of the given queue +

    + + @param parent + Name of the queue whose child-queues' information is needed + @return a list of queue-information for all queues who are direct children + of the given parent queue. + @throws YarnException + @throws IOException]]> +     +     + + + + + + Get information about acls for current user on all the + existing queues. +

    + + @return a list of queue acls ({@link QueueUserACLInfo}) for + current user + @throws YarnException + @throws IOException]]> +     +     + + + + + + + Get a report of the given ApplicationAttempt. +

    + +

    + In secure mode, YARN verifies access to the application, queue + etc. before accepting the request. +

    + + @param applicationAttemptId + {@link ApplicationAttemptId} of the application attempt that needs + a report + @return application attempt report + @throws YarnException + @throws ApplicationAttemptNotFoundException if application attempt + not found + @throws IOException]]> +     +     + + + + + + + Get a report of all (ApplicationAttempts) of Application in the cluster. +

    + + @param applicationId application id of the app + @return a list of reports for all application attempts for specified + application. + @throws YarnException + @throws IOException]]> +     +     + + + + + + + Get a report of the given Container. +

    + +

    + In secure mode, YARN verifies access to the application, queue + etc. before accepting the request. +

    + + @param containerId + {@link ContainerId} of the container that needs a report + @return container report + @throws YarnException + @throws ContainerNotFoundException if container not found. + @throws IOException]]> +     +     + + + + + + + Get a report of all (Containers) of ApplicationAttempt in the cluster. +

    + + @param applicationAttemptId application attempt id + @return a list of reports of all containers for specified application + attempts + @throws YarnException + @throws IOException]]> +     +     + + + + + + + + Attempts to move the given application to the given queue. +

    + + @param appId + Application to move. + @param queue + Queue to place it in to. + @throws YarnException + @throws IOException]]> +     +     + + + + + + Obtain a {@link GetNewReservationResponse} for a new reservation, + which contains the {@link ReservationId} object. +

    + + @return The {@link GetNewReservationResponse} containing a new + {@link ReservationId} object. + @throws YarnException if reservation cannot be created. + @throws IOException if reservation cannot be created.]]> +     +     + + + + + + + The interface used by clients to submit a new reservation to the + {@code ResourceManager}. +

    + +

    + The client packages all details of its request in a + {@link ReservationSubmissionRequest} object. This contains information + about the amount of capacity, temporal constraints, and gang needs. + Furthermore, the reservation might be composed of multiple stages, with + ordering dependencies among them. +

    + +

    + In order to respond, a new admission control component in the + {@code ResourceManager} performs an analysis of the resources that have + been committed over the period of time the user is requesting, verify that + the user requests can be fulfilled, and that it respect a sharing policy + (e.g., {@code CapacityOverTimePolicy}). Once it has positively determined + that the ReservationRequest is satisfiable the {@code ResourceManager} + answers with a {@link ReservationSubmissionResponse} that includes a + {@link ReservationId}. Upon failure to find a valid allocation the response + is an exception with the message detailing the reason of failure. +

    + +

    + The semantics guarantees that the {@link ReservationId} returned, + corresponds to a valid reservation existing in the time-range request by + the user. The amount of capacity dedicated to such reservation can vary + overtime, depending of the allocation that has been determined. But it is + guaranteed to satisfy all the constraint expressed by the user in the + {@link ReservationDefinition} +

    + + @param request request to submit a new Reservation + @return response contains the {@link ReservationId} on accepting the + submission + @throws YarnException if the reservation cannot be created successfully + @throws IOException]]> +     +     + + + + + + + The interface used by clients to update an existing Reservation. This is + referred to as a re-negotiation process, in which a user that has + previously submitted a Reservation. +

    + +

    + The allocation is attempted by virtually substituting all previous + allocations related to this Reservation with new ones, that satisfy the new + {@link ReservationDefinition}. Upon success the previous allocation is + atomically substituted by the new one, and on failure (i.e., if the system + cannot find a valid allocation for the updated request), the previous + allocation remains valid. +

    + + @param request to update an existing Reservation (the + {@link ReservationUpdateRequest} should refer to an existing valid + {@link ReservationId}) + @return response empty on successfully updating the existing reservation + @throws YarnException if the request is invalid or reservation cannot be + updated successfully + @throws IOException]]> +     +     + + + + + + + The interface used by clients to remove an existing Reservation. +

    + + @param request to remove an existing Reservation (the + {@link ReservationDeleteRequest} should refer to an existing valid + {@link ReservationId}) + @return response empty on successfully deleting the existing reservation + @throws YarnException if the request is invalid or reservation cannot be + deleted successfully + @throws IOException]]> +     +     + + + + + + + The interface used by clients to get the list of reservations in a plan. + The reservationId will be used to search for reservations to list if it is + provided. Otherwise, it will select active reservations within the + startTime and endTime (inclusive). +

    + + @param request to list reservations in a plan. Contains fields to select + String queue, ReservationId reservationId, long startTime, + long endTime, and a bool includeReservationAllocations. + + queue: Required. Cannot be null or empty. Refers to the + reservable queue in the scheduler that was selected when + creating a reservation submission + {@link ReservationSubmissionRequest}. + + reservationId: Optional. If provided, other fields will + be ignored. + + startTime: Optional. If provided, only reservations that + end after the startTime will be selected. This defaults + to 0 if an invalid number is used. + + endTime: Optional. If provided, only reservations that + start on or before endTime will be selected. This defaults + to Long.MAX_VALUE if an invalid number is used. + + includeReservationAllocations: Optional. Flag that + determines whether the entire reservation allocations are + to be returned. Reservation allocations are subject to + change in the event of re-planning as described by + {@link ReservationDefinition}. + + @return response that contains information about reservations that are + being searched for. + @throws YarnException if the request is invalid + @throws IOException if the request failed otherwise]]> +     +     + + + + + + The interface used by client to get node to labels mappings in existing cluster +

    + + @return node to labels mappings + @throws YarnException + @throws IOException]]> +     +     + + + + + + The interface used by client to get labels to nodes mapping + in existing cluster +

    + + @return node to labels mappings + @throws YarnException + @throws IOException]]> +     +     + + + + + + + The interface used by client to get labels to nodes mapping + for specified labels in existing cluster +

    + + @param labels labels for which labels to nodes mapping has to be retrieved + @return labels to nodes mappings for specific labels + @throws YarnException + @throws IOException]]> +     +     + + + + + + The interface used by client to get node labels in the cluster +

    + + @return cluster node labels collection + @throws YarnException when there is a failure in + {@link ApplicationClientProtocol} + @throws IOException when there is a failure in + {@link ApplicationClientProtocol}]]> +     +     + + + + + + + + The interface used by client to set priority of an application +

    + @param applicationId + @param priority + @return updated priority of an application. + @throws YarnException + @throws IOException]]> +     +     + + + + + + + + Signal a container identified by given ID. +

    + + @param containerId + {@link ContainerId} of the container that needs to be signaled + @param command the signal container command + @throws YarnException + @throws IOException]]> +     +     +     + + + + + + + + + + + +     + + + + + + + + + + + + + + + + Create a new instance of AMRMClientAsync.

    + + @param intervalMs heartbeat interval in milliseconds between AM and RM + @param callbackHandler callback handler that processes responses from + the ResourceManager]]> +     +     + + + + + + Create a new instance of AMRMClientAsync.

    + + @param client the AMRMClient instance + @param intervalMs heartbeat interval in milliseconds between AM and RM + @param callbackHandler callback handler that processes responses from + the ResourceManager]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + allocate + @param req Resource request]]> + + + + + + + + + + + + + allocate. + Any previous pending resource change request of the same container will be + removed. + + Application that calls this method is expected to maintain the + Containers that are returned from previous successful + allocations or resource changes. By passing in the existing container and a + target resource capability to this method, the application requests the + ResourceManager to change the existing resource allocation to the target + resource allocation. + + @param container The container returned from the last successful resource + allocation or resource change + @param capability The target resource capability of the container]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + check to return true for each 1000 ms. + See also {@link #waitFor(com.google.common.base.Supplier, int)} + and {@link #waitFor(com.google.common.base.Supplier, int, int)} + @param check]]> + + + + + + + + check to return true for each + checkEveryMillis ms. + See also {@link #waitFor(com.google.common.base.Supplier, int, int)} + @param check user defined checker + @param checkEveryMillis interval to call check]]> + + + + + + + + + check to return true for each + checkEveryMillis ms. In the main loop, this method will log + the message "waiting in main loop" for each logInterval times + iteration to confirm the thread is alive. + @param check user defined checker + @param checkEveryMillis interval to call check + @param logInterval interval to log for each]]> + + + + + + + + + + AMRMClientAsync handles communication with the ResourceManager + and provides asynchronous updates on events such as container allocations and + completions. It contains a thread that sends periodic heartbeats to the + ResourceManager. + + It should be used by implementing a CallbackHandler: + 
    + {@code
+ class MyCallbackHandler extends AMRMClientAsync.AbstractCallbackHandler {
+   public void onContainersAllocated(List containers) {
+     [run tasks on the containers]
+   }
+
+   public void onContainersUpdated(List containers) {
+     [determine if resource allocation of containers have been increased in
+      the ResourceManager, and if so, inform the NodeManagers to increase the
+      resource monitor/enforcement on the containers]
+   }
+
+   public void onContainersCompleted(List statuses) {
+     [update progress, check whether app is done]
+   }
+   
+   public void onNodesUpdated(List updated) {}
+   
+   public void onReboot() {}
+ }
+ }
+
    + + The client's lifecycle should be managed similarly to the following: + + 
    + {@code
+ AMRMClientAsync asyncClient = 
+     createAMRMClientAsync(appAttId, 1000, new MyCallbackhandler());
+ asyncClient.init(conf);
+ asyncClient.start();
+ RegisterApplicationMasterResponse response = asyncClient
+    .registerApplicationMaster(appMasterHostname, appMasterRpcPort,
+       appMasterTrackingUrl);
+ asyncClient.addContainerRequest(containerRequest);
+ [... wait for application to complete]
+ asyncClient.unregisterApplicationMaster(status, appMsg, trackingUrl);
+ asyncClient.stop();
+ }
+
    ]]> +     +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + NMClientAsync handles communication with all the NodeManagers + and provides asynchronous updates on getting responses from them. It + maintains a thread pool to communicate with individual NMs where a number of + worker threads process requests to NMs by using {@link NMClientImpl}. The max + size of the thread pool is configurable through + {@link YarnConfiguration#NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE}. + + It should be used in conjunction with a CallbackHandler. For example + + 
    + {@code
+ class MyCallbackHandler extends NMClientAsync.AbstractCallbackHandler {
+   public void onContainerStarted(ContainerId containerId,
+       Map allServiceResponse) {
+     [post process after the container is started, process the response]
+   }
+
+   public void onContainerResourceIncreased(ContainerId containerId,
+       Resource resource) {
+     [post process after the container resource is increased]
+   }
+
+   public void onContainerStatusReceived(ContainerId containerId,
+       ContainerStatus containerStatus) {
+     [make use of the status of the container]
+   }
+
+   public void onContainerStopped(ContainerId containerId) {
+     [post process after the container is stopped]
+   }
+
+   public void onStartContainerError(
+       ContainerId containerId, Throwable t) {
+     [handle the raised exception]
+   }
+
+   public void onGetContainerStatusError(
+       ContainerId containerId, Throwable t) {
+     [handle the raised exception]
+   }
+
+   public void onStopContainerError(
+       ContainerId containerId, Throwable t) {
+     [handle the raised exception]
+   }
+ }
+ }
+
    + + The client's life-cycle should be managed like the following: + + 
    + {@code
+ NMClientAsync asyncClient = 
+     NMClientAsync.createNMClientAsync(new MyCallbackhandler());
+ asyncClient.init(conf);
+ asyncClient.start();
+ asyncClient.startContainer(container, containerLaunchContext);
+ [... wait for container being started]
+ asyncClient.getContainerStatus(container.getId(), container.getNodeId(),
+     container.getContainerToken());
+ [... handle the status in the callback instance]
+ asyncClient.stopContainer(container.getId(), container.getNodeId(),
+     container.getContainerToken());
+ [... wait for container being stopped]
+ asyncClient.stop();
+ }
+
    ]]> +     +     + +     + + + + + + + + + + + + + + + + + + + + + + + + + + +     diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Common_2.8.0.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Common_2.8.0.xml new file mode 100644 index 00000000000..a77dfd5d79c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Common_2.8.0.xml @@ -0,0 +1,2665 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Type of proxy. + @return Proxy to the ResourceManager for the specified client protocol. + @throws IOException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Type information of the proxy + @return Proxy to the RM + @throws IOException]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Send the information of a number of conceptual entities to the timeline + server. It is a blocking API. The method will not return until it gets the + response from the timeline server. +

    + + @param entities + the collection of {@link TimelineEntity} + @return the error information if the sent entities are not correctly stored + @throws IOException + @throws YarnException]]> +     +     + + + + + + + + + Send the information of a number of conceptual entities to the timeline + server. It is a blocking API. The method will not return until it gets the + response from the timeline server. + + This API is only for timeline service v1.5 +

    + + @param appAttemptId {@link ApplicationAttemptId} + @param groupId {@link TimelineEntityGroupId} + @param entities + the collection of {@link TimelineEntity} + @return the error information if the sent entities are not correctly stored + @throws IOException + @throws YarnException]]> +     +     + + + + + + + Send the information of a domain to the timeline server. It is a + blocking API. The method will not return until it gets the response from + the timeline server. +

    + + @param domain + an {@link TimelineDomain} object + @throws IOException + @throws YarnException]]> +     +     + + + + + + + + Send the information of a domain to the timeline server. It is a + blocking API. The method will not return until it gets the response from + the timeline server. + + This API is only for timeline service v1.5 +

    + + @param domain + an {@link TimelineDomain} object + @param appAttemptId {@link ApplicationAttemptId} + @throws IOException + @throws YarnException]]> +     +     + + + + + + + Get a delegation token so as to be able to talk to the timeline server in a + secure way. +

    + + @param renewer + Address of the renewer who can renew these tokens when needed by + securely talking to the timeline server + @return a delegation token ({@link Token}) that can be used to talk to the + timeline server + @throws IOException + @throws YarnException]]> +     +     + + + + + + + Renew a timeline delegation token. +

    + + @param timelineDT + the delegation token to renew + @return the new expiration time + @throws IOException + @throws YarnException]]> +     +     + + + + + + + Cancel a timeline delegation token. +

    + + @param timelineDT + the delegation token to cancel + @throws IOException + @throws YarnException]]> +     +     + + + +     + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + parameterized event of type T]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + InputStream to be checksumed + @return the message digest of the input stream + @throws IOException]]> + + + + + + + + + + + + SharedCacheChecksum object based on the configurable + algorithm implementation + (see yarn.sharedcache.checksum.algo.impl) + + @return SharedCacheChecksum object]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + The object type on which this state machine operates. + @param The state of the entity. + @param The external eventType to be handled. + @param The event object.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Server_Common_2.8.0.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Server_Common_2.8.0.xml new file mode 100644 index 00000000000..1634e2ee418 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/jdiff/Apache_Hadoop_YARN_Server_Common_2.8.0.xml @@ -0,0 +1,829 @@ + + + + + + + + + + + + + + + + + + + + + + + + true if the node is healthy, else false]]> + + + + + diagnostic health report of the node. + @return diagnostic health report of the node]]> + + + + + last timestamp at which the health report was received. + @return last timestamp at which the health report was received]]> + + + + + It includes information such as: +
      +
    • + An indicator of whether the node is healthy, as determined by the + health-check script. +
      • +
    • The previous time at which the health status was reported.
      • +
    • A diagnostic report on the health status.
      • +
    + + @see NodeReport + @see ApplicationClientProtocol#getClusterNodes(org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest)]]> +     +     + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true if the iteration has more elements.]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     From e7b7c96a30b0fee3af22a7719eaaa6481e76a963 Mon Sep 17 00:00:00 2001 From: Junping Du Date: Sat, 25 Mar 2017 14:30:38 -0700 Subject: [PATCH 134/188] Set jdiff.stable.api to 2.8.0 --- hadoop-project-dist/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-project-dist/pom.xml b/hadoop-project-dist/pom.xml index c83b2ffa59c..988d369b4b6 100644 --- a/hadoop-project-dist/pom.xml +++ b/hadoop-project-dist/pom.xml @@ -145,7 +145,7 @@ false - 2.7.2 + 2.8.0 -unstable From b63ae854f9f5a900ca860dcb26c30785af32bde0 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Mon, 27 Mar 2017 11:23:41 +0900 Subject: [PATCH 135/188] YARN-6379. Remove unused argument in ClientRMService. Contributed by Kai Sasaki. --- .../yarn/server/resourcemanager/ClientRMService.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 6130b33736d..8b28d6531c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -1204,7 +1204,7 @@ public class ClientRMService extends AbstractService implements @Override public GetNewReservationResponse getNewReservation( GetNewReservationRequest request) throws YarnException, IOException { - checkReservationSystem(AuditConstants.CREATE_NEW_RESERVATION_REQUEST); + checkReservationSystem(); GetNewReservationResponse response = recordFactory.newRecordInstance(GetNewReservationResponse.class); @@ -1218,7 +1218,7 @@ public class ClientRMService extends AbstractService implements public ReservationSubmissionResponse submitReservation( ReservationSubmissionRequest request) throws YarnException, IOException { // Check if reservation system is enabled - checkReservationSystem(AuditConstants.SUBMIT_RESERVATION_REQUEST); + checkReservationSystem(); ReservationSubmissionResponse response = recordFactory.newRecordInstance(ReservationSubmissionResponse.class); ReservationId reservationId = request.getReservationId(); @@ -1277,7 +1277,7 @@ public class ClientRMService extends AbstractService implements public ReservationUpdateResponse updateReservation( ReservationUpdateRequest request) throws YarnException, IOException { // Check if reservation system is enabled - checkReservationSystem(AuditConstants.UPDATE_RESERVATION_REQUEST); + checkReservationSystem(); ReservationUpdateResponse response = recordFactory.newRecordInstance(ReservationUpdateResponse.class); // Validate the input @@ -1316,7 +1316,7 @@ public class ClientRMService extends AbstractService implements public ReservationDeleteResponse deleteReservation( ReservationDeleteRequest request) throws YarnException, IOException { // Check if reservation system is enabled - checkReservationSystem(AuditConstants.DELETE_RESERVATION_REQUEST); + checkReservationSystem(); ReservationDeleteResponse response = recordFactory.newRecordInstance(ReservationDeleteResponse.class); // Validate the input @@ -1355,7 +1355,7 @@ public class ClientRMService extends AbstractService implements public ReservationListResponse listReservations( ReservationListRequest requestInfo) throws YarnException, IOException { // Check if reservation system is enabled - checkReservationSystem(AuditConstants.LIST_RESERVATION_REQUEST); + checkReservationSystem(); ReservationListResponse response = recordFactory.newRecordInstance(ReservationListResponse.class); @@ -1418,7 +1418,7 @@ public class ClientRMService extends AbstractService implements labelsMgr.getClusterNodeLabels()); } - private void checkReservationSystem(String auditConstant) + private void checkReservationSystem() throws YarnException { // Check if reservation is enabled if (reservationSystem == null) { From e0a2eb71e3eb3efa865068089b649624132d33a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joe=20M=C3=A9sz=C3=A1ros?= Date: Fri, 24 Mar 2017 10:46:31 +0100 Subject: [PATCH 136/188] MAPREDUCE-6866. Fix getNumMapTasks() documentation in JobConf. This closes #205 Signed-off-by: Akira Ajisaka --- .../src/main/java/org/apache/hadoop/mapred/JobConf.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java index ef9ec6168d7..f286a96a44d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java @@ -1286,10 +1286,10 @@ public class JobConf extends Configuration { } /** - * Get configured the number of reduce tasks for this job. + * Get the configured number of map tasks for this job. * Defaults to 1. * - * @return the number of reduce tasks for this job. + * @return the number of map tasks for this job. */ public int getNumMapTasks() { return getInt(JobContext.NUM_MAPS, 1); } @@ -1334,9 +1334,9 @@ public class JobConf extends Configuration { public void setNumMapTasks(int n) { setInt(JobContext.NUM_MAPS, n); } /** - * Get configured the number of reduce tasks for this job. Defaults to + * Get the configured number of reduce tasks for this job. Defaults to * 1. - * + * * @return the number of reduce tasks for this job. */ public int getNumReduceTasks() { return getInt(JobContext.NUM_REDUCES, 1); } From 945b006fe85473c1bb87830dbbfc1df9051c7bb9 Mon Sep 17 00:00:00 2001 From: Junping Du Date: Sun, 26 Mar 2017 23:18:05 -0700 Subject: [PATCH 137/188] MAPREDUCE-6868. License check for jdiff output files should be ignored. Contributed by Akira Ajisaka. --- hadoop-mapreduce-project/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-mapreduce-project/pom.xml b/hadoop-mapreduce-project/pom.xml index 829a56ca164..52c9c074e58 100644 --- a/hadoop-mapreduce-project/pom.xml +++ b/hadoop-mapreduce-project/pom.xml @@ -194,6 +194,7 @@ .eclipse.templates/ lib/jdiff/** + dev-support/jdiff/** From 96e2ab81444f7f87e0df2e1567ae7a26534cd1ab Mon Sep 17 00:00:00 2001 From: Yiqun Lin Date: Mon, 27 Mar 2017 19:23:37 +0800 Subject: [PATCH 138/188] HDFS-11555. Fix typos in class OfflineImageReconstructor. Contributed by Yiqun Lin. --- .../tools/offlineImageViewer/OfflineImageReconstructor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java index e80f4d43e68..d10b9246827 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java @@ -1599,11 +1599,11 @@ class OfflineImageReconstructor { } catch (IOException e) { // Handle the case where does not exist. // Note: fsimage XML files which are missing are also missing - // many other fields that ovi needs to accurately reconstruct the + // many other fields that oiv needs to accurately reconstruct the // fsimage. throw new IOException("No section found at the top of " + "the fsimage XML. This XML file is too old to be processed " + - "by ovi.", e); + "by oiv.", e); } Node version = new Node(); loadNodeChildren(version, "version fields"); From 46d37a65cf09c2714b4c0c4ec0399031d60027a5 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Mon, 27 Mar 2017 05:23:45 -0700 Subject: [PATCH 139/188] HDFS-10506. Addendum patch to include missing changes. Contributed by Akira Ajisaka. --- .../tools/offlineImageViewer/OfflineImageReconstructor.java | 2 +- .../hdfs/tools/offlineImageViewer/PBImageXmlWriter.java | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java index d10b9246827..2b5a19f4865 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java @@ -669,7 +669,7 @@ class OfflineImageReconstructor { throw new IOException(" found without "); } blockBld.setBlockId(id); - Long genstamp = block.removeChildLong(INODE_SECTION_GEMSTAMP); + Long genstamp = block.removeChildLong(INODE_SECTION_GENSTAMP); if (genstamp == null) { throw new IOException(" found without "); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java index 5a42a6b7b02..681ebf6862c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java @@ -119,7 +119,7 @@ public final class PBImageXmlWriter { public static final String INODE_SECTION_PERMISSION = "permission"; public static final String INODE_SECTION_BLOCKS = "blocks"; public static final String INODE_SECTION_BLOCK = "block"; - public static final String INODE_SECTION_GEMSTAMP = "genstamp"; + public static final String INODE_SECTION_GENSTAMP = "genstamp"; public static final String INODE_SECTION_NUM_BYTES = "numBytes"; public static final String INODE_SECTION_FILE_UNDER_CONSTRUCTION = "file-under-construction"; @@ -492,7 +492,7 @@ public final class PBImageXmlWriter { for (BlockProto b : f.getBlocksList()) { out.print("<" + INODE_SECTION_BLOCK + ">"); o(SECTION_ID, b.getBlockId()) - .o(INODE_SECTION_GEMSTAMP, b.getGenStamp()) + .o(INODE_SECTION_GENSTAMP, b.getGenStamp()) .o(INODE_SECTION_NUM_BYTES, b.getNumBytes()); out.print("\n"); } @@ -685,7 +685,7 @@ public final class PBImageXmlWriter { for (BlockProto b : f.getBlocksList()) { out.print("<" + INODE_SECTION_BLOCK + ">"); o(SECTION_ID, b.getBlockId()) - .o(INODE_SECTION_GEMSTAMP, b.getGenStamp()) + .o(INODE_SECTION_GENSTAMP, b.getGenStamp()) .o(INODE_SECTION_NUM_BYTES, b.getNumBytes()); out.print("\n"); } From 858d597be00ce006c1d1d4cd476729e4f1de8d25 Mon Sep 17 00:00:00 2001 From: Jonathan Eagles Date: Mon, 27 Mar 2017 11:01:48 -0500 Subject: [PATCH 140/188] Delay construction of PreCondition.check failure message in Configuration#set (jeagles) --- .../src/main/java/org/apache/hadoop/conf/Configuration.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index a9c8d9cc934..60b03987265 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -1208,7 +1208,7 @@ public class Configuration implements Iterable>, "Property name must not be null"); Preconditions.checkArgument( value != null, - "The value of property " + name + " must not be null"); + "The value of property %s must not be null", name); name = name.trim(); DeprecationContext deprecations = deprecationContext.get(); if (deprecations.getDeprecatedKeyMap().isEmpty()) { From db2adf356a937e4c539b2cf5dccfaf90271cf43e Mon Sep 17 00:00:00 2001 From: Jonathan Eagles Date: Mon, 27 Mar 2017 12:48:44 -0500 Subject: [PATCH 141/188] Configuration#get return value optimization (jeagles) --- .../org/apache/hadoop/conf/Configuration.java | 52 ++++++++++--------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index 60b03987265..ada8b0202e6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -619,42 +619,44 @@ public class Configuration implements Iterable>, * deprecated key, the value of the deprecated key is set as the value for * the provided property name. * + * @param deprecations deprecation context * @param name the property name * @return the first property in the list of properties mapping * the name or the name itself. */ private String[] handleDeprecation(DeprecationContext deprecations, - String name) { + String name) { if (null != name) { name = name.trim(); } - ArrayList names = new ArrayList(); - if (isDeprecated(name)) { - DeprecatedKeyInfo keyInfo = deprecations.getDeprecatedKeyMap().get(name); - if (keyInfo != null) { - if (!keyInfo.getAndSetAccessed()) { - logDeprecation(keyInfo.getWarningMessage(name)); - } - - for (String newKey : keyInfo.newKeys) { - if (newKey != null) { - names.add(newKey); - } + // Initialize the return value with requested name + String[] names = new String[]{name}; + // Deprecated keys are logged once and an updated names are returned + DeprecatedKeyInfo keyInfo = deprecations.getDeprecatedKeyMap().get(name); + if (keyInfo != null) { + if (!keyInfo.getAndSetAccessed()) { + logDeprecation(keyInfo.getWarningMessage(name)); + } + // Override return value for deprecated keys + names = keyInfo.newKeys; + } + // If there are no overlay values we can return early + Properties overlayProperties = getOverlay(); + if (overlayProperties.isEmpty()) { + return names; + } + // Update properties and overlays with reverse lookup values + for (String n : names) { + String deprecatedKey = deprecations.getReverseDeprecatedKeyMap().get(n); + if (deprecatedKey != null && !overlayProperties.containsKey(n)) { + String deprecatedValue = overlayProperties.getProperty(deprecatedKey); + if (deprecatedValue != null) { + getProps().setProperty(n, deprecatedValue); + overlayProperties.setProperty(n, deprecatedValue); } } } - if(names.size() == 0) { - names.add(name); - } - for(String n : names) { - String deprecatedKey = deprecations.getReverseDeprecatedKeyMap().get(n); - if (deprecatedKey != null && !getOverlay().containsKey(n) && - getOverlay().containsKey(deprecatedKey)) { - getProps().setProperty(n, getOverlay().getProperty(deprecatedKey)); - getOverlay().setProperty(n, getOverlay().getProperty(deprecatedKey)); - } - } - return names.toArray(new String[names.size()]); + return names; } private void handleDeprecation() { From cd014d57aa8b896da02b5bcadafbd404bca2bc12 Mon Sep 17 00:00:00 2001 From: Wangda Tan Date: Mon, 27 Mar 2017 13:29:09 -0700 Subject: [PATCH 142/188] YARN-6339. Improve performance for createAndGetApplicationReport. (Yunjiong Zhao via wangda) --- .../yarn/api/records/impl/pb/ProtoUtils.java | 6 ++-- .../resourcemanager/rmapp/RMAppImpl.java | 32 +++++++++++-------- .../TestRMAppLogAggregationStatus.java | 2 ++ 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java index ab283e7c831..926c757f428 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java @@ -296,6 +296,8 @@ public class ProtoUtils { * Log Aggregation Status */ private static final String LOG_AGGREGATION_STATUS_PREFIX = "LOG_"; + private static final int LOG_AGGREGATION_STATUS_PREFIX_LEN = + LOG_AGGREGATION_STATUS_PREFIX.length(); public static LogAggregationStatusProto convertToProtoFormat( LogAggregationStatus e) { return LogAggregationStatusProto.valueOf(LOG_AGGREGATION_STATUS_PREFIX @@ -304,8 +306,8 @@ public class ProtoUtils { public static LogAggregationStatus convertFromProtoFormat( LogAggregationStatusProto e) { - return LogAggregationStatus.valueOf(e.name().replace( - LOG_AGGREGATION_STATUS_PREFIX, "")); + return LogAggregationStatus.valueOf(e.name().substring( + LOG_AGGREGATION_STATUS_PREFIX_LEN)); } /* diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 9f00b2e5fa4..f24908bd5e5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -34,6 +34,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentSkipListSet; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; @@ -177,8 +178,8 @@ public class RMAppImpl implements RMApp, Recoverable { private long logAggregationStartTime = 0; private final long logAggregationStatusTimeout; private final Map logAggregationStatus = - new HashMap(); - private LogAggregationStatus logAggregationStatusForAppReport; + new ConcurrentHashMap(); + private volatile LogAggregationStatus logAggregationStatusForAppReport; private int logAggregationSucceed = 0; private int logAggregationFailed = 0; private Map> logAggregationDiagnosticsForNMs = @@ -1697,26 +1698,23 @@ public class RMAppImpl implements RMApp, Recoverable { public Map getLogAggregationReportsForApp() { try { this.readLock.lock(); - Map outputs = - new HashMap(); - outputs.putAll(logAggregationStatus); - if (!isLogAggregationFinished()) { - for (Entry output : outputs.entrySet()) { + if (!isLogAggregationFinished() && isAppInFinalState(this) && + System.currentTimeMillis() > this.logAggregationStartTime + + this.logAggregationStatusTimeout) { + for (Entry output : + logAggregationStatus.entrySet()) { if (!output.getValue().getLogAggregationStatus() .equals(LogAggregationStatus.TIME_OUT) && !output.getValue().getLogAggregationStatus() .equals(LogAggregationStatus.SUCCEEDED) && !output.getValue().getLogAggregationStatus() - .equals(LogAggregationStatus.FAILED) - && isAppInFinalState(this) - && System.currentTimeMillis() > this.logAggregationStartTime - + this.logAggregationStatusTimeout) { + .equals(LogAggregationStatus.FAILED)) { output.getValue().setLogAggregationStatus( LogAggregationStatus.TIME_OUT); } } } - return outputs; + return Collections.unmodifiableMap(logAggregationStatus); } finally { this.readLock.unlock(); } @@ -1824,11 +1822,17 @@ public class RMAppImpl implements RMApp, Recoverable { // the log aggregation is finished. And the log aggregation status will // not be updated anymore. if (logFailedCount > 0 && isAppInFinalState(this)) { + this.logAggregationStatusForAppReport = + LogAggregationStatus.FAILED; return LogAggregationStatus.FAILED; } else if (logTimeOutCount > 0) { + this.logAggregationStatusForAppReport = + LogAggregationStatus.TIME_OUT; return LogAggregationStatus.TIME_OUT; } if (isAppInFinalState(this)) { + this.logAggregationStatusForAppReport = + LogAggregationStatus.SUCCEEDED; return LogAggregationStatus.SUCCEEDED; } } else if (logRunningWithFailure > 0) { @@ -1844,7 +1848,9 @@ public class RMAppImpl implements RMApp, Recoverable { return this.logAggregationStatusForAppReport .equals(LogAggregationStatus.SUCCEEDED) || this.logAggregationStatusForAppReport - .equals(LogAggregationStatus.FAILED); + .equals(LogAggregationStatus.FAILED) + || this.logAggregationStatusForAppReport + .equals(LogAggregationStatus.TIME_OUT); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java index 55a4eac4eea..677990b8c98 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java @@ -413,6 +413,8 @@ public class TestRMAppLogAggregationStatus { Assert.assertEquals(LogAggregationStatus.TIME_OUT, rmApp.getLogAggregationStatusForAppReport()); + rmApp = (RMAppImpl)createRMApp(conf); + rmApp.handle(new RMAppEvent(rmApp.getApplicationId(), RMAppEventType.KILL)); // If the log aggregation status for all NMs are SUCCEEDED and Application // is at the final state, the log aggregation status for this app will // return SUCCEEDED From 64ea62c1ccc05d9b0a0030beafa60ddd31c38952 Mon Sep 17 00:00:00 2001 From: Masatake Iwasaki Date: Tue, 28 Mar 2017 09:13:44 +0900 Subject: [PATCH 143/188] HDFS-11486. Client close() should not fail fast if the last block is being decommissioned. Contributed by Wei-Chiu Chuang and Yiqun Lin. --- .../apache/hadoop/hdfs/TestDecommission.java | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java index dc0edccdb1a..c2c6be12d1a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java @@ -694,6 +694,56 @@ public class TestDecommission extends AdminStatesBaseTest { assertEquals(dfs.getFileStatus(file).getLen(), writtenBytes); } + + @Test(timeout=120000) + public void testCloseWhileDecommission() throws IOException, + ExecutionException, InterruptedException { + LOG.info("Starting test testCloseWhileDecommission"); + + // min replication = 2 + getConf().setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY, 2); + startCluster(1, 3); + + FileSystem fileSys = getCluster().getFileSystem(0); + FSNamesystem ns = getCluster().getNamesystem(0); + + String openFile = "/testDecommissionWithOpenfile.dat"; + + writeFile(fileSys, new Path(openFile), (short)3); + // make sure the file was open for write + FSDataOutputStream fdos = fileSys.append(new Path(openFile)); + byte[] bytes = new byte[1]; + fdos.write(bytes); + fdos.hsync(); + + LocatedBlocks lbs = NameNodeAdapter.getBlockLocations( + getCluster().getNameNode(0), openFile, 0, fileSize); + + DatanodeInfo[] dnInfos4LastBlock = lbs.getLastLocatedBlock().getLocations(); + + ArrayList nodes = new ArrayList(); + ArrayList dnInfos = new ArrayList(); + + DatanodeManager dm = ns.getBlockManager().getDatanodeManager(); + //decommission 2 of the 3 nodes which have last block + nodes.add(dnInfos4LastBlock[0].getXferAddr()); + dnInfos.add(dm.getDatanode(dnInfos4LastBlock[0])); + nodes.add(dnInfos4LastBlock[1].getXferAddr()); + dnInfos.add(dm.getDatanode(dnInfos4LastBlock[1])); + + // because the cluster has only 3 nodes, and 2 of which are decomm'ed, + // the last block file will remain under replicated. + initExcludeHosts(nodes); + refreshNodes(0); + + // the close() should not fail despite the number of live replicas of + // the last block becomes one. + fdos.close(); + + // make sure the two datanodes remain in decomm in progress state + BlockManagerTestUtil.recheckDecommissionState(dm); + assertTrackedAndPending(dm.getDecomManager(), 2, 0); + } /** * Tests restart of namenode while datanode hosts are added to exclude file From 9bae6720cb8432efd78c909dc624c00e367cedf5 Mon Sep 17 00:00:00 2001 From: Robert Kanter Date: Mon, 27 Mar 2017 17:22:43 -0700 Subject: [PATCH 144/188] YARN-6050. AMs can't be scheduled on racks or nodes (rkanter) --- .../org/apache/hadoop/mapred/YARNRunner.java | 4 +- .../apache/hadoop/mapred/TestYARNRunner.java | 2 +- .../records/ApplicationSubmissionContext.java | 57 +++- .../src/main/proto/yarn_protos.proto | 2 +- .../ApplicationSubmissionContextPBImpl.java | 68 +++- .../nodelabels/CommonNodeLabelsManager.java | 22 ++ .../server/resourcemanager/RMAppManager.java | 104 ++++-- .../server/resourcemanager/RMServerUtils.java | 70 ++++- .../server/resourcemanager/rmapp/RMApp.java | 3 +- .../resourcemanager/rmapp/RMAppImpl.java | 22 +- .../rmapp/attempt/RMAppAttemptImpl.java | 33 +- .../scheduler/AbstractYarnScheduler.java | 5 + .../scheduler/ClusterNodeTracker.java | 65 +++- .../scheduler/ResourceScheduler.java | 9 + .../common/fica/FiCaSchedulerApp.java | 10 +- .../resourcemanager/webapp/dao/AppInfo.java | 2 +- .../yarn/server/resourcemanager/MockRM.java | 45 ++- .../resourcemanager/TestAppManager.java | 195 +++++++++++- .../resourcemanager/TestClientRMService.java | 8 +- .../TestNodeBlacklistingOnAMFailures.java | 184 +++++++++++ .../resourcemanager/TestRMServerUtils.java | 297 ++++++++++++++++++ .../applicationsmanager/MockAsm.java | 6 +- .../metrics/TestSystemMetricsPublisher.java | 3 +- .../resourcemanager/rmapp/MockRMApp.java | 11 +- .../rmapp/TestRMAppTransitions.java | 9 +- .../attempt/TestRMAppAttemptTransitions.java | 19 +- .../capacity/TestApplicationLimits.java | 4 +- .../TestApplicationLimitsByPartition.java | 4 +- .../capacity/TestCapacityScheduler.java | 2 +- .../scheduler/capacity/TestLeafQueue.java | 3 +- .../scheduler/fair/TestFairScheduler.java | 78 +++++ .../webapp/TestRMWebServicesApps.java | 10 +- 32 files changed, 1214 insertions(+), 142 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMServerUtils.java diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java index 228c6af3b4c..2339c79f62b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -589,7 +590,8 @@ public class YARNRunner implements ClientProtocol { amResourceRequest.setCapability(capability); amResourceRequest.setNumContainers(1); amResourceRequest.setNodeLabelExpression(amNodelabelExpression.trim()); - appContext.setAMContainerResourceRequest(amResourceRequest); + appContext.setAMContainerResourceRequests( + Collections.singletonList(amResourceRequest)); } // set labels for the Job containers appContext.setNodeLabelExpression(jobConf diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java index 279c8cec853..c2bda6235a0 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java @@ -571,7 +571,7 @@ public class TestYARNRunner { buildSubmitContext(yarnRunner, jobConf); assertEquals(appSubCtx.getNodeLabelExpression(), "GPU"); - assertEquals(appSubCtx.getAMContainerResourceRequest() + assertEquals(appSubCtx.getAMContainerResourceRequests().get(0) .getNodeLabelExpression(), "highMem"); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java index e562aaae5c3..4f1d147791c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java @@ -18,6 +18,8 @@ package org.apache.hadoop.yarn.api.records; +import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.Set; @@ -100,7 +102,7 @@ public abstract class ApplicationSubmissionContext { amReq.setNumContainers(1); amReq.setRelaxLocality(true); amReq.setNodeLabelExpression(amContainerLabelExpression); - context.setAMContainerResourceRequest(amReq); + context.setAMContainerResourceRequests(Collections.singletonList(amReq)); return context; } @@ -159,7 +161,8 @@ public abstract class ApplicationSubmissionContext { context.setApplicationType(applicationType); context.setKeepContainersAcrossApplicationAttempts(keepContainers); context.setNodeLabelExpression(appLabelExpression); - context.setAMContainerResourceRequest(resourceRequest); + context.setAMContainerResourceRequests( + Collections.singletonList(resourceRequest)); return context; } @@ -454,29 +457,61 @@ public abstract class ApplicationSubmissionContext { public abstract void setNodeLabelExpression(String nodeLabelExpression); /** - * Get ResourceRequest of AM container, if this is not null, scheduler will - * use this to acquire resource for AM container. - * + * Get the ResourceRequest of the AM container. + * + * If this is not null, scheduler will use this to acquire resource for AM + * container. + * * If this is null, scheduler will assemble a ResourceRequest by using * getResource and getPriority of * ApplicationSubmissionContext. - * - * Number of containers and Priority will be ignore. - * - * @return ResourceRequest of AM container + * + * Number of containers and Priority will be ignored. + * + * @return ResourceRequest of the AM container + * @deprecated See {@link #getAMContainerResourceRequests()} */ @Public @Evolving + @Deprecated public abstract ResourceRequest getAMContainerResourceRequest(); /** - * Set ResourceRequest of AM container - * @param request of AM container + * Set ResourceRequest of the AM container + * @param request of the AM container + * @deprecated See {@link #setAMContainerResourceRequests(List)} */ @Public @Evolving + @Deprecated public abstract void setAMContainerResourceRequest(ResourceRequest request); + /** + * Get the ResourceRequests of the AM container. + * + * If this is not null, scheduler will use this to acquire resource for AM + * container. + * + * If this is null, scheduler will use the ResourceRequest as determined by + * getAMContainerResourceRequest and its behavior. + * + * Number of containers and Priority will be ignored. + * + * @return List of ResourceRequests of the AM container + */ + @Public + @Evolving + public abstract List getAMContainerResourceRequests(); + + /** + * Set ResourceRequests of the AM container. + * @param requests of the AM container + */ + @Public + @Evolving + public abstract void setAMContainerResourceRequests( + List requests); + /** * Get the attemptFailuresValidityInterval in milliseconds for the application * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 3b26a5cb3d6..587354a7a72 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -378,7 +378,7 @@ message ApplicationSubmissionContextProto { optional LogAggregationContextProto log_aggregation_context = 14; optional ReservationIdProto reservation_id = 15; optional string node_label_expression = 16; - optional ResourceRequestProto am_container_resource_request = 17; + repeated ResourceRequestProto am_container_resource_request = 17; repeated ApplicationTimeoutMapProto application_timeouts = 18; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java index 66930e74bf8..0148d0e4174 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java @@ -18,6 +18,8 @@ package org.apache.hadoop.yarn.api.records.impl.pb; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -66,7 +68,7 @@ extends ApplicationSubmissionContext { private ContainerLaunchContext amContainer = null; private Resource resource = null; private Set applicationTags = null; - private ResourceRequest amResourceRequest = null; + private List amResourceRequests = null; private LogAggregationContext logAggregationContext = null; private ReservationId reservationId = null; private Map applicationTimeouts = null; @@ -127,9 +129,10 @@ extends ApplicationSubmissionContext { builder.clearApplicationTags(); builder.addAllApplicationTags(this.applicationTags); } - if (this.amResourceRequest != null) { - builder.setAmContainerResourceRequest( - convertToProtoFormat(this.amResourceRequest)); + if (this.amResourceRequests != null) { + builder.clearAmContainerResourceRequest(); + builder.addAllAmContainerResourceRequest( + convertToProtoFormat(this.amResourceRequests)); } if (this.logAggregationContext != null) { builder.setLogAggregationContext( @@ -430,13 +433,23 @@ extends ApplicationSubmissionContext { private PriorityProto convertToProtoFormat(Priority t) { return ((PriorityPBImpl)t).getProto(); } - - private ResourceRequestPBImpl convertFromProtoFormat(ResourceRequestProto p) { - return new ResourceRequestPBImpl(p); + + private List convertFromProtoFormat( + List ps) { + List rs = new ArrayList<>(); + for (ResourceRequestProto p : ps) { + rs.add(new ResourceRequestPBImpl(p)); + } + return rs; } - private ResourceRequestProto convertToProtoFormat(ResourceRequest t) { - return ((ResourceRequestPBImpl)t).getProto(); + private List convertToProtoFormat( + List ts) { + List rs = new ArrayList<>(ts.size()); + for (ResourceRequest t : ts) { + rs.add(((ResourceRequestPBImpl)t).getProto()); + } + return rs; } private ApplicationIdPBImpl convertFromProtoFormat(ApplicationIdProto p) { @@ -485,25 +498,46 @@ extends ApplicationSubmissionContext { } @Override + @Deprecated public ResourceRequest getAMContainerResourceRequest() { - ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder; - if (this.amResourceRequest != null) { - return amResourceRequest; - } // Else via proto - if (!p.hasAmContainerResourceRequest()) { + List reqs = getAMContainerResourceRequests(); + if (reqs == null || reqs.isEmpty()) { return null; } - amResourceRequest = convertFromProtoFormat(p.getAmContainerResourceRequest()); - return amResourceRequest; + return getAMContainerResourceRequests().get(0); } @Override + public List getAMContainerResourceRequests() { + ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder; + if (this.amResourceRequests != null) { + return amResourceRequests; + } // Else via proto + if (p.getAmContainerResourceRequestCount() == 0) { + return null; + } + amResourceRequests = + convertFromProtoFormat(p.getAmContainerResourceRequestList()); + return amResourceRequests; + } + + @Override + @Deprecated public void setAMContainerResourceRequest(ResourceRequest request) { maybeInitBuilder(); if (request == null) { builder.clearAmContainerResourceRequest(); } - this.amResourceRequest = request; + this.amResourceRequests = Collections.singletonList(request); + } + + @Override + public void setAMContainerResourceRequests(List requests) { + maybeInitBuilder(); + if (requests == null) { + builder.clearAmContainerResourceRequest(); + } + this.amResourceRequests = requests; } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java index f3f4ba0ff90..60ade2dbcaa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java @@ -801,6 +801,28 @@ public class CommonNodeLabelsManager extends AbstractService { } } + /** + * Get nodes that have no labels. + * + * @return set of nodes with no labels + */ + public Set getNodesWithoutALabel() { + try { + readLock.lock(); + Set nodes = new HashSet<>(); + for (Host host : nodeCollections.values()) { + for (NodeId nodeId : host.nms.keySet()) { + if (getLabelsByNode(nodeId).isEmpty()) { + nodes.add(nodeId); + } + } + } + return Collections.unmodifiableSet(nodes); + } finally { + readLock.unlock(); + } + } + /** * Get mapping of labels to nodes for all the labels. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 1f446e6d48b..e0cff7bce66 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -17,7 +17,9 @@ */ package org.apache.hadoop.yarn.server.resourcemanager; +import java.util.Collections; import java.util.LinkedList; +import java.util.List; import java.util.Map; import org.apache.commons.logging.Log; @@ -31,6 +33,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; +import org.apache.hadoop.yarn.api.records.ExecutionType; +import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.ResourceRequest; @@ -337,14 +341,16 @@ public class RMAppManager implements EventHandler, // has been disabled. Reject the recovery of this application if it // is true and give clear message so that user can react properly. if (!appContext.getUnmanagedAM() && - application.getAMResourceRequest() == null && + (application.getAMResourceRequests() == null || + application.getAMResourceRequests().isEmpty()) && !YarnConfiguration.areNodeLabelsEnabled(this.conf)) { // check application submission context and see if am resource request // or application itself contains any node label expression. - ResourceRequest amReqFromAppContext = - appContext.getAMContainerResourceRequest(); - String labelExp = (amReqFromAppContext != null) ? - amReqFromAppContext.getNodeLabelExpression() : null; + List amReqsFromAppContext = + appContext.getAMContainerResourceRequests(); + String labelExp = + (amReqsFromAppContext != null && !amReqsFromAppContext.isEmpty()) ? + amReqsFromAppContext.get(0).getNodeLabelExpression() : null; if (labelExp == null) { labelExp = appContext.getNodeLabelExpression(); } @@ -379,9 +385,9 @@ public class RMAppManager implements EventHandler, } ApplicationId applicationId = submissionContext.getApplicationId(); - ResourceRequest amReq = null; + List amReqs = null; try { - amReq = validateAndCreateResourceRequest(submissionContext, isRecovery); + amReqs = validateAndCreateResourceRequest(submissionContext, isRecovery); } catch (InvalidLabelResourceRequestException e) { // This can happen if the application had been submitted and run // with Node Label enabled but recover with Node Label disabled. @@ -444,7 +450,7 @@ public class RMAppManager implements EventHandler, submissionContext.getQueue(), submissionContext, this.scheduler, this.masterService, submitTime, submissionContext.getApplicationType(), - submissionContext.getApplicationTags(), amReq, startTime); + submissionContext.getApplicationTags(), amReqs, startTime); // Concurrent app submissions with same applicationId will fail here // Concurrent app submissions with different applicationIds will not // influence each other @@ -470,7 +476,7 @@ public class RMAppManager implements EventHandler, return application; } - private ResourceRequest validateAndCreateResourceRequest( + private List validateAndCreateResourceRequest( ApplicationSubmissionContext submissionContext, boolean isRecovery) throws InvalidResourceRequestException { // Validation of the ApplicationSubmissionContext needs to be completed @@ -480,33 +486,77 @@ public class RMAppManager implements EventHandler, // Check whether AM resource requirements are within required limits if (!submissionContext.getUnmanagedAM()) { - ResourceRequest amReq = submissionContext.getAMContainerResourceRequest(); - if (amReq == null) { - amReq = BuilderUtils - .newResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY, - ResourceRequest.ANY, submissionContext.getResource(), 1); - } - - // set label expression for AM container - if (null == amReq.getNodeLabelExpression()) { - amReq.setNodeLabelExpression(submissionContext - .getNodeLabelExpression()); + List amReqs = + submissionContext.getAMContainerResourceRequests(); + if (amReqs == null || amReqs.isEmpty()) { + if (submissionContext.getResource() != null) { + amReqs = Collections.singletonList(BuilderUtils + .newResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY, + ResourceRequest.ANY, submissionContext.getResource(), 1)); + } else { + throw new InvalidResourceRequestException("Invalid resource request, " + + "no resources requested"); + } } try { - SchedulerUtils.normalizeAndValidateRequest(amReq, - scheduler.getMaximumResourceCapability(), - submissionContext.getQueue(), scheduler, isRecovery, rmContext); + // Find the ANY request and ensure there's only one + ResourceRequest anyReq = null; + for (ResourceRequest amReq : amReqs) { + if (amReq.getResourceName().equals(ResourceRequest.ANY)) { + if (anyReq == null) { + anyReq = amReq; + } else { + throw new InvalidResourceRequestException("Invalid resource " + + "request, only one resource request with " + + ResourceRequest.ANY + " is allowed"); + } + } + } + if (anyReq == null) { + throw new InvalidResourceRequestException("Invalid resource request, " + + "no resource request specified with " + ResourceRequest.ANY); + } + + // Make sure that all of the requests agree with the ANY request + // and have correct values + for (ResourceRequest amReq : amReqs) { + amReq.setCapability(anyReq.getCapability()); + amReq.setExecutionTypeRequest( + ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED)); + amReq.setNumContainers(1); + amReq.setPriority(RMAppAttemptImpl.AM_CONTAINER_PRIORITY); + } + + // set label expression for AM ANY request if not set + if (null == anyReq.getNodeLabelExpression()) { + anyReq.setNodeLabelExpression(submissionContext + .getNodeLabelExpression()); + } + + // Put ANY request at the front + if (!amReqs.get(0).equals(anyReq)) { + amReqs.remove(anyReq); + amReqs.add(0, anyReq); + } + + // Normalize all requests + for (ResourceRequest amReq : amReqs) { + SchedulerUtils.normalizeAndValidateRequest(amReq, + scheduler.getMaximumResourceCapability(), + submissionContext.getQueue(), scheduler, isRecovery, rmContext); + + amReq.setCapability( + scheduler.getNormalizedResource(amReq.getCapability())); + } + return amReqs; } catch (InvalidResourceRequestException e) { LOG.warn("RM app submission failed in validating AM resource request" + " for application " + submissionContext.getApplicationId(), e); throw e; } - - amReq.setCapability(scheduler.getNormalizedResource(amReq.getCapability())); - return amReq; } - + return null; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java index 0aa7a2c0ec9..35b0c983fac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java @@ -21,13 +21,16 @@ package org.apache.hadoop.yarn.server.resourcemanager; import java.io.IOException; import java.text.ParseException; import java.util.ArrayList; +import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; +import com.google.common.collect.Sets; import org.apache.commons.logging.Log; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.AccessControlException; @@ -41,6 +44,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerUpdateType; import org.apache.hadoop.yarn.api.records.ExecutionType; +import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.Resource; @@ -557,19 +561,65 @@ public class RMServerUtils { * * @param rmContext context * @param conf configuration - * @param amreq am resource request + * @param amReqs am resource requests * @return applicable node count */ public static int getApplicableNodeCountForAM(RMContext rmContext, - Configuration conf, ResourceRequest amreq) { - if (YarnConfiguration.areNodeLabelsEnabled(conf)) { - RMNodeLabelsManager labelManager = rmContext.getNodeLabelManager(); - String amNodeLabelExpression = amreq.getNodeLabelExpression(); - amNodeLabelExpression = (amNodeLabelExpression == null - || amNodeLabelExpression.trim().isEmpty()) - ? RMNodeLabelsManager.NO_LABEL : amNodeLabelExpression; - return labelManager.getActiveNMCountPerLabel(amNodeLabelExpression); + Configuration conf, List amReqs) { + // Determine the list of nodes that are eligible based on the strict + // resource requests + Set nodesForReqs = new HashSet<>(); + for (ResourceRequest amReq : amReqs) { + if (amReq.getRelaxLocality() && + !amReq.getResourceName().equals(ResourceRequest.ANY)) { + nodesForReqs.addAll( + rmContext.getScheduler().getNodeIds(amReq.getResourceName())); + } + } + + if (YarnConfiguration.areNodeLabelsEnabled(conf)) { + // Determine the list of nodes that are eligible based on the node label + String amNodeLabelExpression = amReqs.get(0).getNodeLabelExpression(); + Set nodesForLabels = + getNodeIdsForLabel(rmContext, amNodeLabelExpression); + if (nodesForLabels != null && !nodesForLabels.isEmpty()) { + // If only node labels, strip out any wildcard NodeIds and return + if (nodesForReqs.isEmpty()) { + for (Iterator it = nodesForLabels.iterator(); it.hasNext();) { + if (it.next().getPort() == 0) { + it.remove(); + } + } + return nodesForLabels.size(); + } else { + // The NodeIds common to both the strict resource requests and the + // node label is the eligible set + return Sets.intersection(nodesForReqs, nodesForLabels).size(); + } + } + } + + // If no strict resource request NodeIds nor node label NodeIds, then just + // return the entire cluster + if (nodesForReqs.isEmpty()) { + return rmContext.getScheduler().getNumClusterNodes(); + } + // No node label NodeIds, so return the strict resource request NodeIds + return nodesForReqs.size(); + } + + private static Set getNodeIdsForLabel(RMContext rmContext, + String label) { + label = (label == null || label.trim().isEmpty()) + ? RMNodeLabelsManager.NO_LABEL : label; + if (label.equals(RMNodeLabelsManager.NO_LABEL)) { + // NO_LABEL nodes aren't tracked directly + return rmContext.getNodeLabelManager().getNodesWithoutALabel(); + } else { + Map> labelsToNodes = + rmContext.getNodeLabelManager().getLabelsToNodes( + Collections.singleton(label)); + return labelsToNodes.get(label); } - return rmContext.getScheduler().getNumClusterNodes(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java index b3a87a6e560..43fd1fbfbdf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp; import java.util.Collection; +import java.util.List; import java.util.Map; import java.util.Set; @@ -269,7 +270,7 @@ public interface RMApp extends EventHandler { ReservationId getReservationId(); - ResourceRequest getAMResourceRequest(); + List getAMResourceRequests(); Map getLogAggregationReportsForApp(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index f24908bd5e5..531844c11c3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -196,7 +196,7 @@ public class RMAppImpl implements RMApp, Recoverable { private RMAppEvent eventCausingFinalSaving; private RMAppState targetedFinalState; private RMAppState recoveredFinalState; - private ResourceRequest amReq; + private List amReqs; private CallerContext callerContext; @@ -424,10 +424,10 @@ public class RMAppImpl implements RMApp, Recoverable { ApplicationSubmissionContext submissionContext, YarnScheduler scheduler, ApplicationMasterService masterService, long submitTime, String applicationType, Set applicationTags, - ResourceRequest amReq) { + List amReqs) { this(applicationId, rmContext, config, name, user, queue, submissionContext, scheduler, masterService, submitTime, applicationType, applicationTags, - amReq, -1); + amReqs, -1); } public RMAppImpl(ApplicationId applicationId, RMContext rmContext, @@ -435,7 +435,7 @@ public class RMAppImpl implements RMApp, Recoverable { ApplicationSubmissionContext submissionContext, YarnScheduler scheduler, ApplicationMasterService masterService, long submitTime, String applicationType, Set applicationTags, - ResourceRequest amReq, long startTime) { + List amReqs, long startTime) { this.systemClock = SystemClock.getInstance(); @@ -458,7 +458,7 @@ public class RMAppImpl implements RMApp, Recoverable { } this.applicationType = applicationType; this.applicationTags = applicationTags; - this.amReq = amReq; + this.amReqs = amReqs; if (submissionContext.getPriority() != null) { this.applicationPriority = Priority .newInstance(submissionContext.getPriority().getPriority()); @@ -987,7 +987,7 @@ public class RMAppImpl implements RMApp, Recoverable { if (amBlacklistingEnabled && !submissionContext.getUnmanagedAM()) { currentAMBlacklistManager = new SimpleBlacklistManager( RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, - getAMResourceRequest()), + getAMResourceRequests()), blacklistDisableThreshold); } else { currentAMBlacklistManager = new DisabledBlacklistManager(); @@ -995,7 +995,7 @@ public class RMAppImpl implements RMApp, Recoverable { } RMAppAttempt attempt = new RMAppAttemptImpl(appAttemptId, rmContext, scheduler, masterService, - submissionContext, conf, amReq, this, currentAMBlacklistManager); + submissionContext, conf, amReqs, this, currentAMBlacklistManager); attempts.put(appAttemptId, attempt); currentAttempt = attempt; } @@ -1690,8 +1690,8 @@ public class RMAppImpl implements RMApp, Recoverable { } @Override - public ResourceRequest getAMResourceRequest() { - return this.amReq; + public List getAMResourceRequests() { + return this.amReqs; } @Override @@ -1964,7 +1964,9 @@ public class RMAppImpl implements RMApp, Recoverable { public String getAmNodeLabelExpression() { String amNodeLabelExpression = null; if (!getApplicationSubmissionContext().getUnmanagedAM()) { - amNodeLabelExpression = getAMResourceRequest().getNodeLabelExpression(); + amNodeLabelExpression = + getAMResourceRequests() != null && !getAMResourceRequests().isEmpty() + ? getAMResourceRequests().get(0).getNodeLabelExpression() : null; amNodeLabelExpression = (amNodeLabelExpression == null) ? NodeLabel.NODE_LABEL_EXPRESSION_NOT_SET : amNodeLabelExpression; amNodeLabelExpression = (amNodeLabelExpression.trim().isEmpty()) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 5c0f48e5006..19503e519f2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -192,7 +192,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { private Object transitionTodo; private RMAppAttemptMetrics attemptMetrics = null; - private ResourceRequest amReq = null; + private List amReqs = null; private BlacklistManager blacklistedNodesForAM = null; private String amLaunchDiagnostics; @@ -485,16 +485,16 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { RMContext rmContext, YarnScheduler scheduler, ApplicationMasterService masterService, ApplicationSubmissionContext submissionContext, - Configuration conf, ResourceRequest amReq, RMApp rmApp) { + Configuration conf, List amReqs, RMApp rmApp) { this(appAttemptId, rmContext, scheduler, masterService, submissionContext, - conf, amReq, rmApp, new DisabledBlacklistManager()); + conf, amReqs, rmApp, new DisabledBlacklistManager()); } public RMAppAttemptImpl(ApplicationAttemptId appAttemptId, RMContext rmContext, YarnScheduler scheduler, ApplicationMasterService masterService, ApplicationSubmissionContext submissionContext, - Configuration conf, ResourceRequest amReq, RMApp rmApp, + Configuration conf, List amReqs, RMApp rmApp, BlacklistManager amBlacklistManager) { this.conf = conf; this.applicationAttemptId = appAttemptId; @@ -514,7 +514,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { this.attemptMetrics = new RMAppAttemptMetrics(applicationAttemptId, rmContext); - this.amReq = amReq; + this.amReqs = amReqs; this.blacklistedNodesForAM = amBlacklistManager; final int diagnosticsLimitKC = getDiagnosticsLimitKCOrThrow(conf); @@ -1090,18 +1090,21 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { // will be passed to scheduler, and scheduler will deduct the number after // AM container allocated - // Currently, following fields are all hard code, + // Currently, following fields are all hard coded, // TODO: change these fields when we want to support - // priority/resource-name/relax-locality specification for AM containers - // allocation. - appAttempt.amReq.setNumContainers(1); - appAttempt.amReq.setPriority(AM_CONTAINER_PRIORITY); - appAttempt.amReq.setResourceName(ResourceRequest.ANY); - appAttempt.amReq.setRelaxLocality(true); + // priority or multiple containers AM container allocation. + for (ResourceRequest amReq : appAttempt.amReqs) { + amReq.setNumContainers(1); + amReq.setPriority(AM_CONTAINER_PRIORITY); + } - appAttempt.getAMBlacklistManager().refreshNodeHostCount( + int numNodes = RMServerUtils.getApplicableNodeCountForAM(appAttempt.rmContext, - appAttempt.conf, appAttempt.amReq)); + appAttempt.conf, appAttempt.amReqs); + if (LOG.isDebugEnabled()) { + LOG.debug("Setting node count for blacklist to " + numNodes); + } + appAttempt.getAMBlacklistManager().refreshNodeHostCount(numNodes); ResourceBlacklistRequest amBlacklist = appAttempt.getAMBlacklistManager().getBlacklistUpdates(); @@ -1114,7 +1117,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { Allocation amContainerAllocation = appAttempt.scheduler.allocate( appAttempt.applicationAttemptId, - Collections.singletonList(appAttempt.amReq), + appAttempt.amReqs, EMPTY_CONTAINER_RELEASE_LIST, amBlacklist.getBlacklistAdditions(), amBlacklist.getBlacklistRemovals(), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index 213839dccc9..b954bdf0294 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -1257,4 +1257,9 @@ public abstract class AbstractYarnScheduler rmContainer.getLastConfirmedResource(), null))); } } + + @Override + public List getNodeIds(String resourceName) { + return nodeTracker.getNodeIdsByResourceName(resourceName); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java index e487f698d09..010e64506b6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java @@ -268,6 +268,9 @@ public class ClusterNodeTracker { /** * Convenience method to filter nodes based on a condition. + * + * @param nodeFilter A {@link NodeFilter} for filtering the nodes + * @return A list of filtered nodes */ public List getNodes(NodeFilter nodeFilter) { List nodeList = new ArrayList<>(); @@ -288,6 +291,37 @@ public class ClusterNodeTracker { return nodeList; } + public List getAllNodeIds() { + return getNodeIds(null); + } + + /** + * Convenience method to filter nodes based on a condition. + * + * @param nodeFilter A {@link NodeFilter} for filtering the nodes + * @return A list of filtered nodes + */ + public List getNodeIds(NodeFilter nodeFilter) { + List nodeList = new ArrayList<>(); + readLock.lock(); + try { + if (nodeFilter == null) { + for (N node : nodes.values()) { + nodeList.add(node.getNodeID()); + } + } else { + for (N node : nodes.values()) { + if (nodeFilter.accept(node)) { + nodeList.add(node.getNodeID()); + } + } + } + } finally { + readLock.unlock(); + } + return nodeList; + } + /** * Convenience method to sort nodes. * @@ -320,11 +354,38 @@ public class ClusterNodeTracker { resourceName != null && !resourceName.isEmpty()); List retNodes = new ArrayList<>(); if (ResourceRequest.ANY.equals(resourceName)) { - return getAllNodes(); + retNodes.addAll(getAllNodes()); } else if (nodeNameToNodeMap.containsKey(resourceName)) { retNodes.add(nodeNameToNodeMap.get(resourceName)); } else if (nodesPerRack.containsKey(resourceName)) { - return nodesPerRack.get(resourceName); + retNodes.addAll(nodesPerRack.get(resourceName)); + } else { + LOG.info( + "Could not find a node matching given resourceName " + resourceName); + } + return retNodes; + } + + /** + * Convenience method to return list of {@link NodeId} corresponding to + * resourceName passed in the {@link ResourceRequest}. + * + * @param resourceName Host/rack name of the resource, or + * {@link ResourceRequest#ANY} + * @return list of {@link NodeId} that match the resourceName + */ + public List getNodeIdsByResourceName(final String resourceName) { + Preconditions.checkArgument( + resourceName != null && !resourceName.isEmpty()); + List retNodes = new ArrayList<>(); + if (ResourceRequest.ANY.equals(resourceName)) { + retNodes.addAll(getAllNodeIds()); + } else if (nodeNameToNodeMap.containsKey(resourceName)) { + retNodes.add(nodeNameToNodeMap.get(resourceName).getNodeID()); + } else if (nodesPerRack.containsKey(resourceName)) { + for (N node : nodesPerRack.get(resourceName)) { + retNodes.add(node.getNodeID()); + } } else { LOG.info( "Could not find a node matching given resourceName " + resourceName); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java index 5649ccf7dca..d96d62545c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java @@ -19,10 +19,12 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; import java.io.IOException; +import java.util.List; import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.recovery.Recoverable; @@ -49,4 +51,11 @@ public interface ResourceScheduler extends YarnScheduler, Recoverable { * @throws IOException */ void reinitialize(Configuration conf, RMContext rmContext) throws IOException; + + /** + * Get the {@link NodeId} available in the cluster by resource name. + * @param resourceName resource name + * @return the number of available {@link NodeId} by resource name. + */ + List getNodeIds(String resourceName); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java index fea29bb6a72..5c0b718e33c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java @@ -141,18 +141,20 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt { Resource amResource; String partition; - if (rmApp == null || rmApp.getAMResourceRequest() == null) { + if (rmApp == null || rmApp.getAMResourceRequests() == null + || rmApp.getAMResourceRequests().isEmpty()) { // the rmApp may be undefined (the resource manager checks for this too) // and unmanaged applications do not provide an amResource request // in these cases, provide a default using the scheduler amResource = rmContext.getScheduler().getMinimumResourceCapability(); partition = CommonNodeLabelsManager.NO_LABEL; } else { - amResource = rmApp.getAMResourceRequest().getCapability(); + amResource = rmApp.getAMResourceRequests().get(0).getCapability(); partition = - (rmApp.getAMResourceRequest().getNodeLabelExpression() == null) + (rmApp.getAMResourceRequests().get(0) + .getNodeLabelExpression() == null) ? CommonNodeLabelsManager.NO_LABEL - : rmApp.getAMResourceRequest().getNodeLabelExpression(); + : rmApp.getAMResourceRequests().get(0).getNodeLabelExpression(); } setAppAMNodePartitionName(partition); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 4e85b671da7..10e627a4410 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -229,7 +229,7 @@ public class AppInfo { appNodeLabelExpression = app.getApplicationSubmissionContext().getNodeLabelExpression(); amNodeLabelExpression = (unmanagedApplication) ? null - : app.getAMResourceRequest().getNodeLabelExpression(); + : app.getAMResourceRequests().get(0).getNodeLabelExpression(); // Setting partition based resource usage of application ResourceScheduler scheduler = rm.getRMContext().getScheduler(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java index f9f42ad6e54..aca2fc5ec9c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java @@ -24,6 +24,7 @@ import java.security.PrivilegedAction; import java.security.PrivilegedExceptionAction; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.EnumSet; import java.util.List; import java.util.Map; @@ -678,6 +679,17 @@ public class MockRM extends ResourceManager { tokensConf); } + public RMApp submitApp(List amResourceRequests) + throws Exception { + return submitApp(amResourceRequests, "app1", + "user", null, false, null, + super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, + YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, + false, false, null, 0, null, true, + amResourceRequests.get(0).getPriority(), + amResourceRequests.get(0).getNodeLabelExpression(), null, null); + } + public RMApp submitApp(Resource capability, String name, String user, Map acls, boolean unmanaged, String queue, int maxAppAttempts, Credentials ts, String appType, @@ -688,6 +700,30 @@ public class MockRM extends ResourceManager { Map applicationTimeouts, ByteBuffer tokensConf) throws Exception { + priority = (priority == null) ? Priority.newInstance(0) : priority; + ResourceRequest amResourceRequest = ResourceRequest.newInstance( + priority, ResourceRequest.ANY, capability, 1); + if (amLabel != null && !amLabel.isEmpty()) { + amResourceRequest.setNodeLabelExpression(amLabel.trim()); + } + return submitApp(Collections.singletonList(amResourceRequest), name, user, + acls, unmanaged, queue, maxAppAttempts, ts, appType, waitForAccepted, + keepContainers, isAppIdProvided, applicationId, + attemptFailuresValidityInterval, logAggregationContext, + cancelTokensWhenComplete, priority, amLabel, applicationTimeouts, + tokensConf); + } + + public RMApp submitApp(List amResourceRequests, String name, + String user, Map acls, boolean unmanaged, + String queue, int maxAppAttempts, Credentials ts, String appType, + boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided, + ApplicationId applicationId, long attemptFailuresValidityInterval, + LogAggregationContext logAggregationContext, + boolean cancelTokensWhenComplete, Priority priority, String amLabel, + Map applicationTimeouts, + ByteBuffer tokensConf) + throws Exception { ApplicationId appId = isAppIdProvided ? applicationId : null; ApplicationClientProtocol client = getClientRMService(); if (! isAppIdProvided) { @@ -718,7 +754,6 @@ public class MockRM extends ResourceManager { sub.setApplicationType(appType); ContainerLaunchContext clc = Records .newRecord(ContainerLaunchContext.class); - sub.setResource(capability); clc.setApplicationACLs(acls); if (ts != null && UserGroupInformation.isSecurityEnabled()) { DataOutputBuffer dob = new DataOutputBuffer(); @@ -733,12 +768,12 @@ public class MockRM extends ResourceManager { sub.setLogAggregationContext(logAggregationContext); } sub.setCancelTokensWhenComplete(cancelTokensWhenComplete); - ResourceRequest amResourceRequest = ResourceRequest.newInstance( - Priority.newInstance(0), ResourceRequest.ANY, capability, 1); if (amLabel != null && !amLabel.isEmpty()) { - amResourceRequest.setNodeLabelExpression(amLabel.trim()); + for (ResourceRequest amResourceRequest : amResourceRequests) { + amResourceRequest.setNodeLabelExpression(amLabel.trim()); + } } - sub.setAMContainerResourceRequest(amResourceRequest); + sub.setAMContainerResourceRequests(amResourceRequests); req.setApplicationSubmissionContext(sub); UserGroupInformation fakeUser = UserGroupInformation.createUserForTesting(user, new String[] {"someGroup"}); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index 892f8ba36ec..eb69efa207d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -31,6 +31,8 @@ import static org.mockito.Mockito.when; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.concurrent.ConcurrentMap; @@ -50,6 +52,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.api.records.ExecutionType; +import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; @@ -57,11 +61,13 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter; import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.MockRMApp; @@ -72,6 +78,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; @@ -312,7 +319,7 @@ public class TestAppManager{ ResourceRequest resReg = ResourceRequest.newInstance(Priority.newInstance(0), ResourceRequest.ANY, Resource.newInstance(1024, 1), 1); - sub.setAMContainerResourceRequest(resReg); + sub.setAMContainerResourceRequests(Collections.singletonList(resReg)); req.setApplicationSubmissionContext(sub); sub.setAMContainerSpec(mock(ContainerLaunchContext.class)); try { @@ -522,8 +529,157 @@ public class TestAppManager{ Assert.assertEquals("app event type is wrong before", RMAppEventType.KILL, appEventType); } + @SuppressWarnings("deprecation") @Test - public void testRMAppSubmit() throws Exception { + public void testRMAppSubmitAMContainerResourceRequests() throws Exception { + asContext.setResource(Resources.createResource(1024)); + asContext.setAMContainerResourceRequest( + ResourceRequest.newInstance(Priority.newInstance(0), + ResourceRequest.ANY, Resources.createResource(1024), 1, true)); + List reqs = new ArrayList<>(); + reqs.add(ResourceRequest.newInstance(Priority.newInstance(0), + ResourceRequest.ANY, Resources.createResource(1025), 1, false)); + reqs.add(ResourceRequest.newInstance(Priority.newInstance(0), + "/rack", Resources.createResource(1025), 1, false)); + reqs.add(ResourceRequest.newInstance(Priority.newInstance(0), + "/rack/node", Resources.createResource(1025), 1, true)); + asContext.setAMContainerResourceRequests(cloneResourceRequests(reqs)); + // getAMContainerResourceRequest uses the first entry of + // getAMContainerResourceRequests + Assert.assertEquals(reqs.get(0), asContext.getAMContainerResourceRequest()); + Assert.assertEquals(reqs, asContext.getAMContainerResourceRequests()); + RMApp app = testRMAppSubmit(); + for (ResourceRequest req : reqs) { + req.setNodeLabelExpression(RMNodeLabelsManager.NO_LABEL); + } + // setAMContainerResourceRequests has priority over + // setAMContainerResourceRequest and setResource + Assert.assertEquals(reqs, app.getAMResourceRequests()); + } + + @SuppressWarnings("deprecation") + @Test + public void testRMAppSubmitAMContainerResourceRequest() throws Exception { + asContext.setResource(Resources.createResource(1024)); + asContext.setAMContainerResourceRequests(null); + ResourceRequest req = + ResourceRequest.newInstance(Priority.newInstance(0), + ResourceRequest.ANY, Resources.createResource(1025), 1, true); + asContext.setAMContainerResourceRequest(cloneResourceRequest(req)); + // getAMContainerResourceRequests uses a singleton list of + // getAMContainerResourceRequest + Assert.assertEquals(req, asContext.getAMContainerResourceRequest()); + Assert.assertEquals(req, asContext.getAMContainerResourceRequests().get(0)); + Assert.assertEquals(1, asContext.getAMContainerResourceRequests().size()); + RMApp app = testRMAppSubmit(); + req.setNodeLabelExpression(RMNodeLabelsManager.NO_LABEL); + // setAMContainerResourceRequest has priority over setResource + Assert.assertEquals(Collections.singletonList(req), + app.getAMResourceRequests()); + } + + @Test + public void testRMAppSubmitResource() throws Exception { + asContext.setResource(Resources.createResource(1024)); + asContext.setAMContainerResourceRequests(null); + RMApp app = testRMAppSubmit(); + // setResource + Assert.assertEquals(Collections.singletonList( + ResourceRequest.newInstance(RMAppAttemptImpl.AM_CONTAINER_PRIORITY, + ResourceRequest.ANY, Resources.createResource(1024), 1, true, "")), + app.getAMResourceRequests()); + } + + @Test + public void testRMAppSubmitNoResourceRequests() throws Exception { + asContext.setResource(null); + asContext.setAMContainerResourceRequests(null); + try { + testRMAppSubmit(); + Assert.fail("Should have failed due to no ResourceRequest"); + } catch (InvalidResourceRequestException e) { + Assert.assertEquals( + "Invalid resource request, no resources requested", + e.getMessage()); + } + } + + @Test + public void testRMAppSubmitAMContainerResourceRequestsDisagree() + throws Exception { + asContext.setResource(null); + List reqs = new ArrayList<>(); + ResourceRequest anyReq = ResourceRequest.newInstance( + Priority.newInstance(1), + ResourceRequest.ANY, Resources.createResource(1024), 1, false, "label1", + ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED)); + reqs.add(anyReq); + reqs.add(ResourceRequest.newInstance(Priority.newInstance(2), + "/rack", Resources.createResource(1025), 2, false, "", + ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC))); + reqs.add(ResourceRequest.newInstance(Priority.newInstance(3), + "/rack/node", Resources.createResource(1026), 3, true, "", + ExecutionTypeRequest.newInstance(ExecutionType.OPPORTUNISTIC))); + asContext.setAMContainerResourceRequests(cloneResourceRequests(reqs)); + RMApp app = testRMAppSubmit(); + // It should force the requests to all agree on these points + for (ResourceRequest req : reqs) { + req.setCapability(anyReq.getCapability()); + req.setExecutionTypeRequest( + ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED)); + req.setNumContainers(1); + req.setPriority(Priority.newInstance(0)); + } + Assert.assertEquals(reqs, app.getAMResourceRequests()); + } + + @Test + public void testRMAppSubmitAMContainerResourceRequestsNoAny() + throws Exception { + asContext.setResource(null); + List reqs = new ArrayList<>(); + reqs.add(ResourceRequest.newInstance(Priority.newInstance(1), + "/rack", Resources.createResource(1025), 1, false)); + reqs.add(ResourceRequest.newInstance(Priority.newInstance(1), + "/rack/node", Resources.createResource(1025), 1, true)); + asContext.setAMContainerResourceRequests(cloneResourceRequests(reqs)); + // getAMContainerResourceRequest uses the first entry of + // getAMContainerResourceRequests + Assert.assertEquals(reqs, asContext.getAMContainerResourceRequests()); + try { + testRMAppSubmit(); + Assert.fail("Should have failed due to missing ANY ResourceRequest"); + } catch (InvalidResourceRequestException e) { + Assert.assertEquals( + "Invalid resource request, no resource request specified with *", + e.getMessage()); + } + } + + @Test + public void testRMAppSubmitAMContainerResourceRequestsTwoManyAny() + throws Exception { + asContext.setResource(null); + List reqs = new ArrayList<>(); + reqs.add(ResourceRequest.newInstance(Priority.newInstance(1), + ResourceRequest.ANY, Resources.createResource(1025), 1, false)); + reqs.add(ResourceRequest.newInstance(Priority.newInstance(1), + ResourceRequest.ANY, Resources.createResource(1025), 1, false)); + asContext.setAMContainerResourceRequests(cloneResourceRequests(reqs)); + // getAMContainerResourceRequest uses the first entry of + // getAMContainerResourceRequests + Assert.assertEquals(reqs, asContext.getAMContainerResourceRequests()); + try { + testRMAppSubmit(); + Assert.fail("Should have failed due to too many ANY ResourceRequests"); + } catch (InvalidResourceRequestException e) { + Assert.assertEquals( + "Invalid resource request, only one resource request with * is " + + "allowed", e.getMessage()); + } + } + + private RMApp testRMAppSubmit() throws Exception { appMonitor.submitApplication(asContext, "test"); RMApp app = rmContext.getRMApps().get(appId); Assert.assertNotNull("app is null", app); @@ -534,12 +690,14 @@ public class TestAppManager{ // wait for event to be processed int timeoutSecs = 0; - while ((getAppEventType() == RMAppEventType.KILL) && + while ((getAppEventType() == RMAppEventType.KILL) && timeoutSecs++ < 20) { Thread.sleep(1000); } Assert.assertEquals("app event type sent is wrong", RMAppEventType.START, getAppEventType()); + + return app; } @Test @@ -737,6 +895,15 @@ public class TestAppManager{ ResourceCalculator rs = mock(ResourceCalculator.class); when(scheduler.getResourceCalculator()).thenReturn(rs); + when(scheduler.getNormalizedResource(any())) + .thenAnswer(new Answer() { + @Override + public Resource answer(InvocationOnMock invocationOnMock) + throws Throwable { + return (Resource) invocationOnMock.getArguments()[0]; + } + }); + return scheduler; } @@ -753,4 +920,26 @@ public class TestAppManager{ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); } + private static ResourceRequest cloneResourceRequest(ResourceRequest req) { + return ResourceRequest.newInstance( + Priority.newInstance(req.getPriority().getPriority()), + new String(req.getResourceName()), + Resource.newInstance(req.getCapability().getMemorySize(), + req.getCapability().getVirtualCores()), + req.getNumContainers(), + req.getRelaxLocality(), + req.getNodeLabelExpression() != null + ? new String(req.getNodeLabelExpression()) : null, + ExecutionTypeRequest.newInstance( + req.getExecutionTypeRequest().getExecutionType())); + } + + private static List cloneResourceRequests( + List reqs) { + List cloneReqs = new ArrayList<>(); + for (ResourceRequest req : reqs) { + cloneReqs.add(cloneResourceRequest(req)); + } + return cloneReqs; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 7a67aa87594..f0e60a246f3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -38,6 +38,7 @@ import java.security.AccessControlException; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; @@ -1307,9 +1308,9 @@ public class TestClientRMService { spy(new RMAppImpl(applicationId3, rmContext, config, null, null, queueName, asContext, yarnScheduler, null, System.currentTimeMillis(), "YARN", null, - BuilderUtils.newResourceRequest( + Collections.singletonList(BuilderUtils.newResourceRequest( RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, - Resource.newInstance(1024, 1), 1)){ + Resource.newInstance(1024, 1), 1))){ @Override public ApplicationReport createAndGetApplicationReport( String clientUserName, boolean allowAccess) { @@ -1323,7 +1324,8 @@ public class TestClientRMService { return report; } }); - app.getAMResourceRequest().setNodeLabelExpression(amNodeLabelExpression); + app.getAMResourceRequests().get(0) + .setNodeLabelExpression(amNodeLabelExpression); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance( ApplicationId.newInstance(123456, 1), 1); RMAppAttemptImpl rmAppAttemptImpl = spy(new RMAppAttemptImpl(attemptId, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java index c80a799e7d8..b4adf480b35 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager; +import java.util.ArrayList; import java.util.List; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -28,6 +29,9 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.DrainDispatcher; @@ -156,6 +160,186 @@ public class TestNodeBlacklistingOnAMFailures { currentNode.getNodeId(), allocatedContainers.get(0).getNodeId()); } + @Test(timeout = 100000) + public void testNodeBlacklistingOnAMFailureStrictNodeLocality() + throws Exception { + YarnConfiguration conf = new YarnConfiguration(); + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + conf.setBoolean(YarnConfiguration.AM_SCHEDULING_NODE_BLACKLISTING_ENABLED, + true); + + DrainDispatcher dispatcher = new DrainDispatcher(); + MockRM rm = startRM(conf, dispatcher); + CapacityScheduler scheduler = (CapacityScheduler) rm.getResourceScheduler(); + + // Register 5 nodes, so that we can blacklist atleast one if AM container + // is failed. As per calculation it will be like, 5nodes * 0.2 (default)=1. + MockNM nm1 = + new MockNM("127.0.0.1:1234", 8000, rm.getResourceTrackerService()); + nm1.registerNode(); + + MockNM nm2 = + new MockNM("127.0.0.2:2345", 8000, rm.getResourceTrackerService()); + nm2.registerNode(); + + MockNM nm3 = + new MockNM("127.0.0.3:2345", 8000, rm.getResourceTrackerService()); + nm3.registerNode(); + + MockNM nm4 = + new MockNM("127.0.0.4:2345", 8000, rm.getResourceTrackerService()); + nm4.registerNode(); + + MockNM nm5 = + new MockNM("127.0.0.5:2345", 8000, rm.getResourceTrackerService()); + nm5.registerNode(); + + // Specify a strict locality on nm2 + List reqs = new ArrayList<>(); + ResourceRequest nodeReq = ResourceRequest.newInstance( + Priority.newInstance(0), nm2.getNodeId().getHost(), + Resource.newInstance(200, 1), 1, true); + ResourceRequest rackReq = ResourceRequest.newInstance( + Priority.newInstance(0), "/default-rack", + Resource.newInstance(200, 1), 1, false); + ResourceRequest anyReq = ResourceRequest.newInstance( + Priority.newInstance(0), ResourceRequest.ANY, + Resource.newInstance(200, 1), 1, false); + reqs.add(anyReq); + reqs.add(rackReq); + reqs.add(nodeReq); + RMApp app = rm.submitApp(reqs); + + MockAM am1 = MockRM.launchAndRegisterAM(app, rm, nm2); + ContainerId amContainerId = + ContainerId.newContainerId(am1.getApplicationAttemptId(), 1); + RMContainer rmContainer = scheduler.getRMContainer(amContainerId); + NodeId nodeWhereAMRan = rmContainer.getAllocatedNode(); + Assert.assertEquals(nm2.getNodeId(), nodeWhereAMRan); + + // Set the exist status to INVALID so that we can verify that the system + // automatically blacklisting the node + makeAMContainerExit(rm, amContainerId, nm2, ContainerExitStatus.INVALID); + + // restart the am + RMAppAttempt attempt = MockRM.waitForAttemptScheduled(app, rm); + System.out.println("New AppAttempt launched " + attempt.getAppAttemptId()); + + nm2.nodeHeartbeat(true); + dispatcher.await(); + + // Now the AM container should be allocated + MockRM.waitForState(attempt, RMAppAttemptState.ALLOCATED, 20000); + + MockAM am2 = rm.sendAMLaunched(attempt.getAppAttemptId()); + rm.waitForState(attempt.getAppAttemptId(), RMAppAttemptState.LAUNCHED); + amContainerId = + ContainerId.newContainerId(am2.getApplicationAttemptId(), 1); + rmContainer = scheduler.getRMContainer(amContainerId); + nodeWhereAMRan = rmContainer.getAllocatedNode(); + + // The second AM should be on the same node because the strict locality + // made the eligible nodes only 1, so the blacklisting threshold kicked in + System.out.println("AM ran on " + nodeWhereAMRan); + Assert.assertEquals(nm2.getNodeId(), nodeWhereAMRan); + + am2.registerAppAttempt(); + rm.waitForState(app.getApplicationId(), RMAppState.RUNNING); + } + + @Test(timeout = 100000) + public void testNodeBlacklistingOnAMFailureRelaxedNodeLocality() + throws Exception { + YarnConfiguration conf = new YarnConfiguration(); + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + conf.setBoolean(YarnConfiguration.AM_SCHEDULING_NODE_BLACKLISTING_ENABLED, + true); + + DrainDispatcher dispatcher = new DrainDispatcher(); + MockRM rm = startRM(conf, dispatcher); + CapacityScheduler scheduler = (CapacityScheduler) rm.getResourceScheduler(); + + // Register 5 nodes, so that we can blacklist atleast one if AM container + // is failed. As per calculation it will be like, 5nodes * 0.2 (default)=1. + MockNM nm1 = + new MockNM("127.0.0.1:1234", 8000, rm.getResourceTrackerService()); + nm1.registerNode(); + + MockNM nm2 = + new MockNM("127.0.0.2:2345", 8000, rm.getResourceTrackerService()); + nm2.registerNode(); + + MockNM nm3 = + new MockNM("127.0.0.3:2345", 8000, rm.getResourceTrackerService()); + nm3.registerNode(); + + MockNM nm4 = + new MockNM("127.0.0.4:2345", 8000, rm.getResourceTrackerService()); + nm4.registerNode(); + + MockNM nm5 = + new MockNM("127.0.0.5:2345", 8000, rm.getResourceTrackerService()); + nm5.registerNode(); + + // Specify a relaxed locality on nm2 + List reqs = new ArrayList<>(); + ResourceRequest nodeReq = ResourceRequest.newInstance( + Priority.newInstance(0), nm2.getNodeId().getHost(), + Resource.newInstance(200, 1), 1, true); + ResourceRequest rackReq = ResourceRequest.newInstance( + Priority.newInstance(0), "/default-rack", + Resource.newInstance(200, 1), 1, true); + ResourceRequest anyReq = ResourceRequest.newInstance( + Priority.newInstance(0), ResourceRequest.ANY, + Resource.newInstance(200, 1), 1, true); + reqs.add(anyReq); + reqs.add(rackReq); + reqs.add(nodeReq); + RMApp app = rm.submitApp(reqs); + + MockAM am1 = MockRM.launchAndRegisterAM(app, rm, nm2); + ContainerId amContainerId = + ContainerId.newContainerId(am1.getApplicationAttemptId(), 1); + RMContainer rmContainer = scheduler.getRMContainer(amContainerId); + NodeId nodeWhereAMRan = rmContainer.getAllocatedNode(); + Assert.assertEquals(nm2.getNodeId(), nodeWhereAMRan); + + // Set the exist status to INVALID so that we can verify that the system + // automatically blacklisting the node + makeAMContainerExit(rm, amContainerId, nm2, ContainerExitStatus.INVALID); + + // restart the am + RMAppAttempt attempt = MockRM.waitForAttemptScheduled(app, rm); + System.out.println("New AppAttempt launched " + attempt.getAppAttemptId()); + + nm2.nodeHeartbeat(true); + nm1.nodeHeartbeat(true); + nm3.nodeHeartbeat(true); + nm4.nodeHeartbeat(true); + nm5.nodeHeartbeat(true); + dispatcher.await(); + + // Now the AM container should be allocated + MockRM.waitForState(attempt, RMAppAttemptState.ALLOCATED, 20000); + + MockAM am2 = rm.sendAMLaunched(attempt.getAppAttemptId()); + rm.waitForState(attempt.getAppAttemptId(), RMAppAttemptState.LAUNCHED); + amContainerId = + ContainerId.newContainerId(am2.getApplicationAttemptId(), 1); + rmContainer = scheduler.getRMContainer(amContainerId); + nodeWhereAMRan = rmContainer.getAllocatedNode(); + + // The second AM should be on a different node because the relaxed locality + // made the app schedulable on other nodes and nm2 is blacklisted + System.out.println("AM ran on " + nodeWhereAMRan); + Assert.assertNotEquals(nm2.getNodeId(), nodeWhereAMRan); + + am2.registerAppAttempt(); + rm.waitForState(app.getApplicationId(), RMAppState.RUNNING); + } + @Test(timeout = 100000) public void testNoBlacklistingForNonSystemErrors() throws Exception { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMServerUtils.java new file mode 100644 index 00000000000..078b8fd3290 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMServerUtils.java @@ -0,0 +1,297 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager; + +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class TestRMServerUtils { + @Test + public void testGetApplicableNodeCountForAMLocality() throws Exception { + List rack1Nodes = new ArrayList<>(); + for (int i = 0; i < 29; i++) { + rack1Nodes.add(NodeId.newInstance("host" + i, 1234)); + } + NodeId node1 = NodeId.newInstance("node1", 1234); + NodeId node2 = NodeId.newInstance("node2", 1234); + rack1Nodes.add(node2); + + YarnConfiguration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, false); + ResourceScheduler scheduler = Mockito.mock(ResourceScheduler.class); + Mockito.when(scheduler.getNumClusterNodes()).thenReturn(100); + Mockito.when(scheduler.getNodeIds("/rack1")).thenReturn(rack1Nodes); + Mockito.when(scheduler.getNodeIds("node1")) + .thenReturn(Collections.singletonList(node1)); + Mockito.when(scheduler.getNodeIds("node2")) + .thenReturn(Collections.singletonList(node2)); + RMContext rmContext = Mockito.mock(RMContext.class); + Mockito.when(rmContext.getScheduler()).thenReturn(scheduler); + + ResourceRequest anyReq = createResourceRequest(ResourceRequest.ANY, + true, null); + List reqs = new ArrayList<>(); + reqs.add(anyReq); + Assert.assertEquals(100, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + + ResourceRequest rackReq = createResourceRequest("/rack1", true, null); + reqs.add(rackReq); + Assert.assertEquals(30, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + anyReq.setRelaxLocality(false); + Assert.assertEquals(30, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + rackReq.setRelaxLocality(false); + Assert.assertEquals(100, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + + ResourceRequest node1Req = createResourceRequest("node1", false, null); + reqs.add(node1Req); + Assert.assertEquals(100, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + node1Req.setRelaxLocality(true); + Assert.assertEquals(1, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + rackReq.setRelaxLocality(true); + Assert.assertEquals(31, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + + ResourceRequest node2Req = createResourceRequest("node2", false, null); + reqs.add(node2Req); + Assert.assertEquals(31, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + node2Req.setRelaxLocality(true); + Assert.assertEquals(31, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + rackReq.setRelaxLocality(false); + Assert.assertEquals(2, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + node1Req.setRelaxLocality(false); + Assert.assertEquals(1, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + node2Req.setRelaxLocality(false); + Assert.assertEquals(100, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + } + + @Test + public void testGetApplicableNodeCountForAMLabels() throws Exception { + Set noLabelNodes = new HashSet<>(); + for (int i = 0; i < 80; i++) { + noLabelNodes.add(NodeId.newInstance("host" + i, 1234)); + } + Set label1Nodes = new HashSet<>(); + for (int i = 80; i < 90; i++) { + label1Nodes.add(NodeId.newInstance("host" + i, 1234)); + } + label1Nodes.add(NodeId.newInstance("host101", 0)); + label1Nodes.add(NodeId.newInstance("host102", 0)); + Map> label1NodesMap = new HashMap<>(); + label1NodesMap.put("label1", label1Nodes); + + YarnConfiguration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true); + ResourceScheduler scheduler = Mockito.mock(ResourceScheduler.class); + Mockito.when(scheduler.getNumClusterNodes()).thenReturn(100); + RMContext rmContext = Mockito.mock(RMContext.class); + Mockito.when(rmContext.getScheduler()).thenReturn(scheduler); + RMNodeLabelsManager labMan = Mockito.mock(RMNodeLabelsManager.class); + Mockito.when(labMan.getNodesWithoutALabel()).thenReturn(noLabelNodes); + Mockito.when(labMan.getLabelsToNodes(Collections.singleton("label1"))) + .thenReturn(label1NodesMap); + Mockito.when(rmContext.getNodeLabelManager()).thenReturn(labMan); + + ResourceRequest anyReq = createResourceRequest(ResourceRequest.ANY, + true, null); + List reqs = new ArrayList<>(); + reqs.add(anyReq); + Assert.assertEquals(80, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + anyReq.setNodeLabelExpression("label1"); + Assert.assertEquals(10, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + } + + @Test + public void testGetApplicableNodeCountForAMLocalityAndLabels() + throws Exception { + List rack1Nodes = new ArrayList<>(); + for (int i = 0; i < 29; i++) { + rack1Nodes.add(NodeId.newInstance("host" + i, 1234)); + } + NodeId node1 = NodeId.newInstance("node1", 1234); + NodeId node2 = NodeId.newInstance("node2", 1234); + rack1Nodes.add(node2); + Set noLabelNodes = new HashSet<>(); + for (int i = 0; i < 19; i++) { + noLabelNodes.add(rack1Nodes.get(i)); + } + noLabelNodes.add(node2); + for (int i = 29; i < 89; i++) { + noLabelNodes.add(NodeId.newInstance("host" + i, 1234)); + } + Set label1Nodes = new HashSet<>(); + label1Nodes.add(node1); + for (int i = 89; i < 93; i++) { + label1Nodes.add(NodeId.newInstance("host" + i, 1234)); + } + for (int i = 19; i < 29; i++) { + label1Nodes.add(rack1Nodes.get(i)); + } + label1Nodes.add(NodeId.newInstance("host101", 0)); + label1Nodes.add(NodeId.newInstance("host102", 0)); + Map> label1NodesMap = new HashMap<>(); + label1NodesMap.put("label1", label1Nodes); + + YarnConfiguration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true); + ResourceScheduler scheduler = Mockito.mock(ResourceScheduler.class); + Mockito.when(scheduler.getNumClusterNodes()).thenReturn(100); + Mockito.when(scheduler.getNodeIds("/rack1")).thenReturn(rack1Nodes); + Mockito.when(scheduler.getNodeIds("node1")) + .thenReturn(Collections.singletonList(node1)); + Mockito.when(scheduler.getNodeIds("node2")) + .thenReturn(Collections.singletonList(node2)); + RMContext rmContext = Mockito.mock(RMContext.class); + Mockito.when(rmContext.getScheduler()).thenReturn(scheduler); + RMNodeLabelsManager labMan = Mockito.mock(RMNodeLabelsManager.class); + Mockito.when(labMan.getNodesWithoutALabel()).thenReturn(noLabelNodes); + Mockito.when(labMan.getLabelsToNodes(Collections.singleton("label1"))) + .thenReturn(label1NodesMap); + Mockito.when(rmContext.getNodeLabelManager()).thenReturn(labMan); + + ResourceRequest anyReq = createResourceRequest(ResourceRequest.ANY, + true, null); + List reqs = new ArrayList<>(); + reqs.add(anyReq); + Assert.assertEquals(80, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + + ResourceRequest rackReq = createResourceRequest("/rack1", true, null); + reqs.add(rackReq); + Assert.assertEquals(20, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + anyReq.setRelaxLocality(false); + Assert.assertEquals(20, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + rackReq.setRelaxLocality(false); + Assert.assertEquals(80, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + + ResourceRequest node1Req = createResourceRequest("node1", false, null); + reqs.add(node1Req); + Assert.assertEquals(80, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + node1Req.setRelaxLocality(true); + Assert.assertEquals(0, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + rackReq.setRelaxLocality(true); + Assert.assertEquals(20, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + + ResourceRequest node2Req = createResourceRequest("node2", false, null); + reqs.add(node2Req); + Assert.assertEquals(20, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + node2Req.setRelaxLocality(true); + Assert.assertEquals(20, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + rackReq.setRelaxLocality(false); + Assert.assertEquals(1, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + node1Req.setRelaxLocality(false); + Assert.assertEquals(1, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + node2Req.setRelaxLocality(false); + Assert.assertEquals(80, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + + anyReq.setNodeLabelExpression("label1"); + rackReq.setNodeLabelExpression("label1"); + node1Req.setNodeLabelExpression("label1"); + node2Req.setNodeLabelExpression("label1"); + anyReq.setRelaxLocality(true); + reqs = new ArrayList<>(); + reqs.add(anyReq); + Assert.assertEquals(15, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + + rackReq.setRelaxLocality(true); + reqs.add(rackReq); + Assert.assertEquals(10, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + anyReq.setRelaxLocality(false); + Assert.assertEquals(10, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + rackReq.setRelaxLocality(false); + Assert.assertEquals(15, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + + node1Req.setRelaxLocality(false); + reqs.add(node1Req); + Assert.assertEquals(15, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + node1Req.setRelaxLocality(true); + Assert.assertEquals(1, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + rackReq.setRelaxLocality(true); + Assert.assertEquals(11, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + + node2Req.setRelaxLocality(false); + reqs.add(node2Req); + Assert.assertEquals(11, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + node2Req.setRelaxLocality(true); + Assert.assertEquals(11, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + rackReq.setRelaxLocality(false); + Assert.assertEquals(1, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + node1Req.setRelaxLocality(false); + Assert.assertEquals(0, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + node2Req.setRelaxLocality(false); + Assert.assertEquals(15, + RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); + } + + private ResourceRequest createResourceRequest(String resource, + boolean relaxLocality, String nodeLabel) { + return ResourceRequest.newInstance(Priority.newInstance(0), + resource, Resource.newInstance(1, 1), 1, relaxLocality, nodeLabel); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index 9be52c6d3b6..5246eb79a15 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -57,7 +57,7 @@ import com.google.common.collect.Lists; public abstract class MockAsm extends MockApps { public static class ApplicationBase implements RMApp { - ResourceRequest amReq; + List amReqs; @Override public String getUser() { throw new UnsupportedOperationException("Not supported yet."); @@ -204,8 +204,8 @@ public abstract class MockAsm extends MockApps { } @Override - public ResourceRequest getAMResourceRequest() { - return this.amReq; + public List getAMResourceRequests() { + return this.amReqs; } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java index 55e93c1e6cb..7005bca6585 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java @@ -526,7 +526,8 @@ public class TestSystemMetricsPublisher { when(app.getAppNodeLabelExpression()).thenCallRealMethod(); ResourceRequest amReq = mock(ResourceRequest.class); when(amReq.getNodeLabelExpression()).thenReturn("high-mem"); - when(app.getAMResourceRequest()).thenReturn(amReq); + when(app.getAMResourceRequests()) + .thenReturn(Collections.singletonList(amReq)); when(app.getAmNodeLabelExpression()).thenCallRealMethod(); when(app.getApplicationPriority()).thenReturn(Priority.newInstance(10)); when(app.getCallerContext()) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java index 118b6bc7489..9290ff8faa0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp; import java.util.Collection; import java.util.Collections; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Set; @@ -62,14 +63,14 @@ public class MockRMApp implements RMApp { StringBuilder diagnostics = new StringBuilder(); RMAppAttempt attempt; int maxAppAttempts = 1; - ResourceRequest amReq; + List amReqs; public MockRMApp(int newid, long time, RMAppState newState) { finish = time; id = MockApps.newAppID(newid); state = newState; - amReq = ResourceRequest.newInstance(Priority.UNDEFINED, "0.0.0.0", - Resource.newInstance(0, 0), 1); + amReqs = Collections.singletonList(ResourceRequest.newInstance( + Priority.UNDEFINED, "0.0.0.0", Resource.newInstance(0, 0), 1)); } public MockRMApp(int newid, long time, RMAppState newState, String userName) { @@ -276,8 +277,8 @@ public class MockRMApp implements RMApp { } @Override - public ResourceRequest getAMResourceRequest() { - return this.amReq; + public List getAMResourceRequests() { + return this.amReqs; } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index 488485112cb..5aa7af9e519 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -30,8 +30,10 @@ import static org.mockito.Mockito.verify; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.Map; import org.apache.commons.logging.Log; @@ -271,7 +273,8 @@ public class TestRMAppTransitions { submissionContext.setAMContainerSpec(mock(ContainerLaunchContext.class)); RMApp application = new RMAppImpl(applicationId, rmContext, conf, name, user, queue, submissionContext, scheduler, masterService, - System.currentTimeMillis(), "YARN", null, mock(ResourceRequest.class)); + System.currentTimeMillis(), "YARN", null, + new ArrayList()); testAppStartState(applicationId, user, name, queue, application); this.rmContext.getRMApps().putIfAbsent(application.getApplicationId(), @@ -1024,9 +1027,9 @@ public class TestRMAppTransitions { submissionContext.getQueue(), submissionContext, scheduler, null, appState.getSubmitTime(), submissionContext.getApplicationType(), submissionContext.getApplicationTags(), - BuilderUtils.newResourceRequest( + Collections.singletonList(BuilderUtils.newResourceRequest( RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, - submissionContext.getResource(), 1)); + submissionContext.getResource(), 1))); Assert.assertEquals(RMAppState.NEW, application.getState()); RMAppEvent recoverEvent = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index ced5bd9b8d8..9a4b6dc7f72 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -328,9 +328,9 @@ public class TestRMAppAttemptTransitions { applicationAttempt = new RMAppAttemptImpl(applicationAttemptId, spyRMContext, scheduler, masterService, submissionContext, new Configuration(), - BuilderUtils.newResourceRequest( + Collections.singletonList(BuilderUtils.newResourceRequest( RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, - submissionContext.getResource(), 1), application); + submissionContext.getResource(), 1)), application); when(application.getCurrentAppAttempt()).thenReturn(applicationAttempt); when(application.getApplicationId()).thenReturn(applicationId); @@ -1108,9 +1108,9 @@ public class TestRMAppAttemptTransitions { new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(), spyRMContext, scheduler,masterService, submissionContext, myConf, - BuilderUtils.newResourceRequest( + Collections.singletonList(BuilderUtils.newResourceRequest( RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, - submissionContext.getResource(), 1), application); + submissionContext.getResource(), 1)), application); //submit, schedule and allocate app attempt myApplicationAttempt.handle( @@ -1584,9 +1584,9 @@ public class TestRMAppAttemptTransitions { applicationAttempt = new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(), spyRMContext, scheduler, masterService, submissionContext, new Configuration(), - BuilderUtils.newResourceRequest( + Collections.singletonList(BuilderUtils.newResourceRequest( RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, - submissionContext.getResource(), 1), application); + submissionContext.getResource(), 1)), application); when(submissionContext.getKeepContainersAcrossApplicationAttempts()) .thenReturn(true); when(submissionContext.getMaxAppAttempts()).thenReturn(1); @@ -1645,9 +1645,10 @@ public class TestRMAppAttemptTransitions { applicationAttempt = new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(), spyRMContext, scheduler, masterService, submissionContext, - new Configuration(), ResourceRequest.newInstance( - Priority.UNDEFINED, "host1", Resource.newInstance(3333, 1), 3, - false, "label-expression"), application); + new Configuration(), Collections.singletonList( + ResourceRequest.newInstance(Priority.UNDEFINED, "host1", + Resource.newInstance(3333, 1), 3, + false, "label-expression")), application); new RMAppAttemptImpl.ScheduleTransition().transition( (RMAppAttemptImpl) applicationAttempt, null); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java index bb0a1239fca..8aca235f8aa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java @@ -28,6 +28,7 @@ import static org.mockito.Mockito.when; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -613,7 +614,8 @@ public class TestApplicationLimits { ResourceRequest amResourceRequest = mock(ResourceRequest.class); Resource amResource = Resources.createResource(0, 0); when(amResourceRequest.getCapability()).thenReturn(amResource); - when(rmApp.getAMResourceRequest()).thenReturn(amResourceRequest); + when(rmApp.getAMResourceRequests()).thenReturn( + Collections.singletonList(amResourceRequest)); Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId)Matchers.any()); when(spyRMContext.getRMApps()).thenReturn(spyApps); RMAppAttempt rmAppAttempt = mock(RMAppAttempt.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java index b70a359f591..0aac2ef23da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java @@ -25,6 +25,7 @@ import static org.mockito.Mockito.when; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -639,7 +640,8 @@ public class TestApplicationLimitsByPartition { ResourceRequest amResourceRequest = mock(ResourceRequest.class); Resource amResource = Resources.createResource(0, 0); when(amResourceRequest.getCapability()).thenReturn(amResource); - when(rmApp.getAMResourceRequest()).thenReturn(amResourceRequest); + when(rmApp.getAMResourceRequests()).thenReturn( + Collections.singletonList(amResourceRequest)); Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId) Matchers.any()); when(spyRMContext.getRMApps()).thenReturn(spyApps); RMAppAttempt rmAppAttempt = mock(RMAppAttempt.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index 293bac21174..ff0f7cfc377 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -3205,7 +3205,7 @@ public class TestCapacityScheduler { RMApp rmApp = rm.submitApp(amMemory, "app-1", "user_0", null, queueName); assertEquals("RMApp does not containes minimum allocation", - minAllocResource, rmApp.getAMResourceRequest().getCapability()); + minAllocResource, rmApp.getAMResourceRequests().get(0).getCapability()); ResourceScheduler scheduler = rm.getRMContext().getScheduler(); LeafQueue queueA = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 3fbbae3f59a..1162b9fc7b4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -151,7 +151,8 @@ public class TestLeafQueue { amResourceRequest = mock(ResourceRequest.class); when(amResourceRequest.getCapability()).thenReturn( Resources.createResource(0, 0)); - when(rmApp.getAMResourceRequest()).thenReturn(amResourceRequest); + when(rmApp.getAMResourceRequests()).thenReturn( + Collections.singletonList(amResourceRequest)); Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId)Matchers.any()); when(spyRMContext.getRMApps()).thenReturn(spyApps); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index baf74346fed..9bf79f6100d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -3204,6 +3204,84 @@ public class TestFairScheduler extends FairSchedulerTestBase { assertEquals(1, app.getLiveContainers().size()); } + @Test + public void testAMStrictLocalityRack() throws IOException { + testAMStrictLocality(false, false); + } + + @Test + public void testAMStrictLocalityNode() throws IOException { + testAMStrictLocality(true, false); + } + + @Test + public void testAMStrictLocalityRackInvalid() throws IOException { + testAMStrictLocality(false, true); + } + + @Test + public void testAMStrictLocalityNodeInvalid() throws IOException { + testAMStrictLocality(true, true); + } + + private void testAMStrictLocality(boolean node, boolean invalid) + throws IOException { + scheduler.init(conf); + scheduler.start(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024), 1, + "127.0.0.1"); + NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); + scheduler.handle(nodeEvent1); + + RMNode node2 = MockNodes.newNodeInfo(2, Resources.createResource(1024), 2, + "127.0.0.2"); + NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2); + scheduler.handle(nodeEvent2); + + List reqs = new ArrayList<>(); + ResourceRequest nodeRequest = createResourceRequest(1024, + node2.getHostName(), 1, 1, true); + if (node && invalid) { + nodeRequest.setResourceName("invalid"); + } + ResourceRequest rackRequest = createResourceRequest(1024, + node2.getRackName(), 1, 1, !node); + if (!node && invalid) { + rackRequest.setResourceName("invalid"); + } + ResourceRequest anyRequest = createResourceRequest(1024, + ResourceRequest.ANY, 1, 1, false); + reqs.add(anyRequest); + reqs.add(rackRequest); + if (node) { + reqs.add(nodeRequest); + } + + ApplicationAttemptId attId1 = + createSchedulingRequest("queue1", "user1", reqs); + + scheduler.update(); + + NodeUpdateSchedulerEvent node2UpdateEvent = + new NodeUpdateSchedulerEvent(node2); + + FSAppAttempt app = scheduler.getSchedulerApp(attId1); + + // node2 should get the container + scheduler.handle(node2UpdateEvent); + if (invalid) { + assertEquals(0, app.getLiveContainers().size()); + assertEquals(0, scheduler.getNode(node2.getNodeID()).getNumContainers()); + assertEquals(0, scheduler.getNode(node1.getNodeID()).getNumContainers()); + } else { + assertEquals(1, app.getLiveContainers().size()); + assertEquals(1, scheduler.getNode(node2.getNodeID()).getNumContainers()); + assertEquals(0, scheduler.getNode(node1.getNodeID()).getNumContainers()); + } + } + /** * Strict locality requests shouldn't reserve resources on another node. */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 30f25e950e3..fb9e8edfb0e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1213,8 +1213,8 @@ public class TestRMWebServicesApps extends JerseyTestBase { assertEquals(app1.getApplicationId().toString(), appInfo.getAppId()); assertEquals(app1.getName(), appInfo.getName()); assertEquals(app1.createApplicationState(), appInfo.getState()); - assertEquals(app1.getAMResourceRequest().getCapability().getMemorySize(), - appInfo.getAllocatedMB()); + assertEquals(app1.getAMResourceRequests().get(0).getCapability() + .getMemorySize(), appInfo.getAllocatedMB()); rm.stop(); } @@ -1427,7 +1427,7 @@ public class TestRMWebServicesApps extends JerseyTestBase { expectedNumberOfElements++; appNodeLabelExpression = info.getString("appNodeLabelExpression"); } - if (app.getAMResourceRequest().getNodeLabelExpression() != null) { + if (app.getAMResourceRequests().get(0).getNodeLabelExpression() != null) { expectedNumberOfElements++; amNodeLabelExpression = info.getString("amNodeLabelExpression"); } @@ -1534,7 +1534,7 @@ public class TestRMWebServicesApps extends JerseyTestBase { app.getApplicationSubmissionContext().getNodeLabelExpression(), appNodeLabelExpression); assertEquals("unmanagedApplication doesn't match", - app.getAMResourceRequest().getNodeLabelExpression(), + app.getAMResourceRequests().get(0).getNodeLabelExpression(), amNodeLabelExpression); assertEquals("amRPCAddress", AppInfo.getAmRPCAddressFromRMAppAttempt(app.getCurrentAppAttempt()), @@ -1561,7 +1561,7 @@ public class TestRMWebServicesApps extends JerseyTestBase { String nodeLabelExpression, int numContainers, boolean relaxLocality, int priority, String resourceName, long memory, long vCores, String executionType, boolean enforceExecutionType) { - ResourceRequest request = app.getAMResourceRequest(); + ResourceRequest request = app.getAMResourceRequests().get(0); assertEquals("nodeLabelExpression doesn't match", request.getNodeLabelExpression(), nodeLabelExpression); assertEquals("numContainers doesn't match", request.getNumContainers(), From 448ec81fd7133f413853570f116d5f6e16f68bd9 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Tue, 28 Mar 2017 14:32:40 +0900 Subject: [PATCH 145/188] HADOOP-14218. Replace assertThat with assertTrue in MetricsAsserts. --- .../test/java/org/apache/hadoop/test/MetricsAsserts.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MetricsAsserts.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MetricsAsserts.java index a7bbe841366..b2f6054fe85 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MetricsAsserts.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MetricsAsserts.java @@ -27,7 +27,6 @@ import static org.mockito.AdditionalMatchers.geq; import static org.mockito.Mockito.*; import org.mockito.stubbing.Answer; -import org.mockito.internal.matchers.GreaterThan; import org.mockito.invocation.InvocationOnMock; import org.mockito.ArgumentCaptor; @@ -329,8 +328,8 @@ public class MetricsAsserts { */ public static void assertCounterGt(String name, long greater, MetricsRecordBuilder rb) { - Assert.assertThat("Bad value for metric " + name, getLongCounter(name, rb), - new GreaterThan(greater)); + Assert.assertTrue("Bad value for metric " + name, + getLongCounter(name, rb) > greater); } /** @@ -352,8 +351,8 @@ public class MetricsAsserts { */ public static void assertGaugeGt(String name, double greater, MetricsRecordBuilder rb) { - Assert.assertThat("Bad value for metric " + name, getDoubleGauge(name, rb), - new GreaterThan(greater)); + Assert.assertTrue("Bad value for metric " + name, + getDoubleGauge(name, rb) > greater); } /** From 253e3e78abb9ebe0e8103d25538ff07875b33c84 Mon Sep 17 00:00:00 2001 From: John Zhuge Date: Mon, 20 Feb 2017 21:27:46 -0800 Subject: [PATCH 146/188] HADOOP-14038. Rename ADLS credential properties. Contributed by John Zhuge. --- .../org/apache/hadoop/conf/Configuration.java | 15 +++- .../org/apache/hadoop/fs/adl/AdlConfKeys.java | 38 +++++++-- .../apache/hadoop/fs/adl/AdlFileSystem.java | 4 + .../src/site/markdown/index.md | 22 ++--- .../fs/adl/TestAzureADTokenProvider.java | 5 +- .../fs/adl/TestValidateConfiguration.java | 83 +++++++++++++++++-- 6 files changed, 138 insertions(+), 29 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index ada8b0202e6..da4f6ac6b3f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -751,7 +751,20 @@ public class Configuration implements Iterable>, this.loadDefaults = other.loadDefaults; setQuietMode(other.getQuietMode()); } - + + /** + * Reload existing configuration instances. + */ + public static synchronized void reloadExistingConfigurations() { + if (LOG.isDebugEnabled()) { + LOG.debug("Reloading " + REGISTRY.keySet().size() + + " existing configurations"); + } + for (Configuration conf : REGISTRY.keySet()) { + conf.reloadConfiguration(); + } + } + /** * Add a default resource. Resources are loaded in the order of the resources * added. diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java index 8fc8e00b84c..d3a5565e6d5 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java +++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java @@ -20,6 +20,8 @@ package org.apache.hadoop.fs.adl; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configuration.DeprecationDelta; /** * Constants. @@ -28,25 +30,25 @@ import org.apache.hadoop.classification.InterfaceStability; @InterfaceStability.Evolving public final class AdlConfKeys { // OAuth2 Common Configuration - public static final String AZURE_AD_REFRESH_URL_KEY = "dfs.adls.oauth2" - + ".refresh.url"; + public static final String AZURE_AD_REFRESH_URL_KEY = + "fs.adl.oauth2.refresh.url"; // optional when provider type is refresh or client id. public static final String AZURE_AD_TOKEN_PROVIDER_CLASS_KEY = - "dfs.adls.oauth2.access.token.provider"; + "fs.adl.oauth2.access.token.provider"; public static final String AZURE_AD_CLIENT_ID_KEY = - "dfs.adls.oauth2.client.id"; + "fs.adl.oauth2.client.id"; public static final String AZURE_AD_TOKEN_PROVIDER_TYPE_KEY = - "dfs.adls.oauth2.access.token.provider.type"; + "fs.adl.oauth2.access.token.provider.type"; // OAuth Refresh Token Configuration public static final String AZURE_AD_REFRESH_TOKEN_KEY = - "dfs.adls.oauth2.refresh.token"; + "fs.adl.oauth2.refresh.token"; public static final String TOKEN_PROVIDER_TYPE_REFRESH_TOKEN = "RefreshToken"; // OAuth Client Cred Token Configuration public static final String AZURE_AD_CLIENT_SECRET_KEY = - "dfs.adls.oauth2.credential"; + "fs.adl.oauth2.credential"; public static final String TOKEN_PROVIDER_TYPE_CLIENT_CRED = "ClientCredential"; @@ -75,7 +77,7 @@ public final class AdlConfKeys { static final int DEFAULT_WRITE_AHEAD_BUFFER_SIZE = 4 * 1024 * 1024; static final String LATENCY_TRACKER_KEY = - "adl.dfs.enable.client.latency.tracker"; + "adl.enable.client.latency.tracker"; static final boolean LATENCY_TRACKER_DEFAULT = true; static final String ADL_EXPERIMENT_POSITIONAL_READ_KEY = @@ -90,6 +92,26 @@ public final class AdlConfKeys { "adl.feature.ownerandgroup.enableupn"; static final boolean ADL_ENABLEUPN_FOR_OWNERGROUP_DEFAULT = false; + public static void addDeprecatedKeys() { + Configuration.addDeprecations(new DeprecationDelta[]{ + new DeprecationDelta("dfs.adls.oauth2.access.token.provider.type", + AZURE_AD_TOKEN_PROVIDER_TYPE_KEY), + new DeprecationDelta("dfs.adls.oauth2.client.id", + AZURE_AD_CLIENT_ID_KEY), + new DeprecationDelta("dfs.adls.oauth2.refresh.token", + AZURE_AD_REFRESH_TOKEN_KEY), + new DeprecationDelta("dfs.adls.oauth2.refresh.url", + AZURE_AD_REFRESH_URL_KEY), + new DeprecationDelta("dfs.adls.oauth2.credential", + AZURE_AD_CLIENT_SECRET_KEY), + new DeprecationDelta("dfs.adls.oauth2.access.token.provider", + AZURE_AD_TOKEN_PROVIDER_CLASS_KEY), + new DeprecationDelta("adl.dfs.enable.client.latency.tracker", + LATENCY_TRACKER_KEY) + }); + Configuration.reloadExistingConfigurations(); + } + private AdlConfKeys() { } } diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java index e0e273e6f21..0b860b36592 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java +++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java @@ -88,6 +88,10 @@ public class AdlFileSystem extends FileSystem { private AccessTokenProvider tokenProvider; private AzureADTokenProvider azureTokenProvider; + static { + AdlConfKeys.addDeprecatedKeys(); + } + @Override public String getScheme() { return SCHEME; diff --git a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md index de8fda25d17..d2da858dfc7 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md +++ b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md @@ -83,7 +83,7 @@ Add the following properties to the cluster's `core-site.xml` ```xml - dfs.adls.oauth2.access.token.provider.type + fs.adl.oauth2.access.token.provider.type RefreshToken ``` @@ -95,12 +95,12 @@ service associated with the client id. See [*Active Directory Library For Java*] ```xml - dfs.adls.oauth2.client.id + fs.adl.oauth2.client.id - dfs.adls.oauth2.refresh.token + fs.adl.oauth2.refresh.token ``` @@ -133,22 +133,22 @@ Add the following properties to your `core-site.xml` ```xml - dfs.adls.oauth2.access.token.provider.type + fs.adl.oauth2.access.token.provider.type ClientCredential - dfs.adls.oauth2.refresh.url + fs.adl.oauth2.refresh.url TOKEN ENDPOINT FROM STEP 7 ABOVE - dfs.adls.oauth2.client.id + fs.adl.oauth2.client.id CLIENT ID FROM STEP 7 ABOVE - dfs.adls.oauth2.credential + fs.adl.oauth2.credential PASSWORD FROM STEP 7 ABOVE ``` @@ -166,9 +166,9 @@ For additional reading on the credential provider API, see ##### Provisioning ```bash -hadoop credential create dfs.adls.oauth2.client.id -value 123 +hadoop credential create fs.adl.oauth2.client.id -value 123 -provider localjceks://file/home/foo/adls.jceks -hadoop credential create dfs.adls.oauth2.refresh.token -value 123 +hadoop credential create fs.adl.oauth2.refresh.token -value 123 -provider localjceks://file/home/foo/adls.jceks ``` @@ -176,7 +176,7 @@ hadoop credential create dfs.adls.oauth2.refresh.token -value 123 ```xml - dfs.adls.oauth2.access.token.provider.type + fs.adl.oauth2.access.token.provider.type RefreshToken @@ -190,7 +190,7 @@ hadoop credential create dfs.adls.oauth2.refresh.token -value 123 ```bash hadoop distcp - [-D dfs.adls.oauth2.access.token.provider.type=RefreshToken + [-D fs.adl.oauth2.access.token.provider.type=RefreshToken -D hadoop.security.credential.provider.path=localjceks://file/home/user/adls.jceks] hdfs://:9001/user/foo/srcDir adl://.azuredatalakestore.net/tgtDir/ diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java index 70f2a7f992f..3867e74fa77 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java @@ -46,6 +46,7 @@ import static org.junit.Assert.assertEquals; import org.apache.hadoop.security.ProviderUtils; import org.apache.hadoop.security.alias.CredentialProvider; import org.apache.hadoop.security.alias.CredentialProviderFactory; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -121,8 +122,8 @@ public class TestAzureADTokenProvider { Assert.fail("Initialization should have failed due no token provider " + "configuration"); } catch (IllegalArgumentException e) { - Assert.assertTrue( - e.getMessage().contains("dfs.adls.oauth2.access.token.provider")); + GenericTestUtils.assertExceptionContains( + AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, e); } conf.setClass(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, CustomMockTokenProvider.class, AzureADTokenProvider.class); diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java index 4cabaa3d08d..3d51b42111e 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java @@ -18,6 +18,8 @@ package org.apache.hadoop.fs.adl; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.Assert; import org.junit.Test; @@ -56,6 +58,11 @@ import static org.apache.hadoop.fs.adl.AdlConfKeys .TOKEN_PROVIDER_TYPE_REFRESH_TOKEN; import static org.apache.hadoop.fs.adl.AdlConfKeys.WRITE_BUFFER_SIZE_KEY; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; + /** * Validate configuration keys defined for adl storage file system instance. */ @@ -64,18 +71,18 @@ public class TestValidateConfiguration { @Test public void validateConfigurationKeys() { Assert - .assertEquals("dfs.adls.oauth2.refresh.url", AZURE_AD_REFRESH_URL_KEY); - Assert.assertEquals("dfs.adls.oauth2.access.token.provider", + .assertEquals("fs.adl.oauth2.refresh.url", AZURE_AD_REFRESH_URL_KEY); + Assert.assertEquals("fs.adl.oauth2.access.token.provider", AZURE_AD_TOKEN_PROVIDER_CLASS_KEY); - Assert.assertEquals("dfs.adls.oauth2.client.id", AZURE_AD_CLIENT_ID_KEY); - Assert.assertEquals("dfs.adls.oauth2.refresh.token", + Assert.assertEquals("fs.adl.oauth2.client.id", AZURE_AD_CLIENT_ID_KEY); + Assert.assertEquals("fs.adl.oauth2.refresh.token", AZURE_AD_REFRESH_TOKEN_KEY); Assert - .assertEquals("dfs.adls.oauth2.credential", AZURE_AD_CLIENT_SECRET_KEY); + .assertEquals("fs.adl.oauth2.credential", AZURE_AD_CLIENT_SECRET_KEY); Assert.assertEquals("adl.debug.override.localuserasfileowner", ADL_DEBUG_OVERRIDE_LOCAL_USER_AS_OWNER); - Assert.assertEquals("dfs.adls.oauth2.access.token.provider.type", + Assert.assertEquals("fs.adl.oauth2.access.token.provider.type", AZURE_AD_TOKEN_PROVIDER_TYPE_KEY); Assert.assertEquals("adl.feature.client.cache.readahead", @@ -88,7 +95,7 @@ public class TestValidateConfiguration { Assert.assertEquals("ClientCredential", TOKEN_PROVIDER_TYPE_CLIENT_CRED); - Assert.assertEquals("adl.dfs.enable.client.latency.tracker", + Assert.assertEquals("adl.enable.client.latency.tracker", LATENCY_TRACKER_KEY); Assert.assertEquals(true, LATENCY_TRACKER_DEFAULT); @@ -109,4 +116,66 @@ public class TestValidateConfiguration { Assert.assertEquals(false, ADL_ENABLEUPN_FOR_OWNERGROUP_DEFAULT); } + + @Test + public void testSetDeprecatedKeys() throws ClassNotFoundException { + Configuration conf = new Configuration(true); + setDeprecatedKeys(conf); + + // Force AdlFileSystem static initialization to register deprecated keys. + Class.forName(AdlFileSystem.class.getName()); + + assertDeprecatedKeys(conf); + } + + @Test + public void testLoadDeprecatedKeys() + throws IOException, ClassNotFoundException { + Configuration saveConf = new Configuration(false); + setDeprecatedKeys(saveConf); + + final File testRootDir = GenericTestUtils.getTestDir(); + File confXml = new File(testRootDir, "testLoadDeprecatedKeys.xml"); + OutputStream out = new FileOutputStream(confXml); + saveConf.writeXml(out); + out.close(); + + Configuration conf = new Configuration(true); + conf.addResource(confXml.toURI().toURL()); + + // Trigger loading the configuration resources by getting any key. + conf.get("dummy.key"); + + // Force AdlFileSystem static initialization to register deprecated keys. + Class.forName(AdlFileSystem.class.getName()); + + assertDeprecatedKeys(conf); + } + + private void setDeprecatedKeys(Configuration conf) { + conf.set("dfs.adls.oauth2.access.token.provider.type", "dummyType"); + conf.set("dfs.adls.oauth2.client.id", "dummyClientId"); + conf.set("dfs.adls.oauth2.refresh.token", "dummyRefreshToken"); + conf.set("dfs.adls.oauth2.refresh.url", "dummyRefreshUrl"); + conf.set("dfs.adls.oauth2.credential", "dummyCredential"); + conf.set("dfs.adls.oauth2.access.token.provider", "dummyClass"); + conf.set("adl.dfs.enable.client.latency.tracker", "dummyTracker"); + } + + private void assertDeprecatedKeys(Configuration conf) { + Assert.assertEquals("dummyType", + conf.get(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY)); + Assert.assertEquals("dummyClientId", + conf.get(AZURE_AD_CLIENT_ID_KEY)); + Assert.assertEquals("dummyRefreshToken", + conf.get(AZURE_AD_REFRESH_TOKEN_KEY)); + Assert.assertEquals("dummyRefreshUrl", + conf.get(AZURE_AD_REFRESH_URL_KEY)); + Assert.assertEquals("dummyCredential", + conf.get(AZURE_AD_CLIENT_SECRET_KEY)); + Assert.assertEquals("dummyClass", + conf.get(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY)); + Assert.assertEquals("dummyTracker", + conf.get(LATENCY_TRACKER_KEY)); + } } From fdf8f8ebca9987a1956ce464fe33ea6a3ad28d72 Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Tue, 28 Mar 2017 09:33:26 -0500 Subject: [PATCH 147/188] YARN-6359. TestRM#testApplicationKillAtAcceptedState fails rarely due to race condition. Contributed by Robert Kanter --- .../hadoop/yarn/server/resourcemanager/TestRM.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java index cdf582e1f21..39313d06bd4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java @@ -18,6 +18,8 @@ package org.apache.hadoop.yarn.server.resourcemanager; +import com.google.common.base.Supplier; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.yarn.event.DrainDispatcher; import org.junit.Before; import static org.mockito.Matchers.argThat; @@ -631,7 +633,13 @@ public class TestRM extends ParameterizedSchedulerTestBase { rm.waitForState(application.getApplicationId(), RMAppState.KILLED); // test metrics - metrics = rm.getResourceScheduler().getRootQueueMetrics(); + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + return appsKilled + 1 == metrics.getAppsKilled() + && appsSubmitted + 1 == metrics.getAppsSubmitted(); + } + }, 100, 10000); Assert.assertEquals(appsKilled + 1, metrics.getAppsKilled()); Assert.assertEquals(appsSubmitted + 1, metrics.getAppsSubmitted()); } From 6b0933643835d7696ced011cfdb8b74f63022e8b Mon Sep 17 00:00:00 2001 From: Yiqun Lin Date: Tue, 28 Mar 2017 23:02:07 +0800 Subject: [PATCH 148/188] HDFS-11577. Combine the old and the new chooseRandom for better performance. Contributed by Chen Liang. --- .../apache/hadoop/net/NetworkTopology.java | 2 +- .../hadoop/hdfs/net/DFSNetworkTopology.java | 76 +++++++++++++++++-- .../hdfs/net/TestDFSNetworkTopology.java | 57 ++++++++++++++ 3 files changed, 126 insertions(+), 9 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java index 051012b79df..f8cecddaeb8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java @@ -496,7 +496,7 @@ public class NetworkTopology { } } - private Node chooseRandom(final String scope, String excludedScope, + protected Node chooseRandom(final String scope, String excludedScope, final Collection excludedNodes) { if (excludedScope != null) { if (scope.startsWith(excludedScope)) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java index ee83dba0095..259e2759dbd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java @@ -36,7 +36,6 @@ import java.util.Random; * remaining parts should be the same. * * Currently a placeholder to test storage type info. - * TODO : add "chooseRandom with storageType info" function. */ public class DFSNetworkTopology extends NetworkTopology { @@ -56,6 +55,7 @@ public class DFSNetworkTopology extends NetworkTopology { * * @param scope range of nodes from which a node will be chosen * @param excludedNodes nodes to be excluded from + * @param type the storage type we search for * @return the chosen node */ public Node chooseRandomWithStorageType(final String scope, @@ -74,6 +74,69 @@ public class DFSNetworkTopology extends NetworkTopology { } } + /** + * Randomly choose one node from scope with the given storage type. + * + * If scope starts with ~, choose one from the all nodes except for the + * ones in scope; otherwise, choose one from scope. + * If excludedNodes is given, choose a node that's not in excludedNodes. + * + * This call would make up to two calls. It first tries to get a random node + * (with old method) and check if it satisfies. If yes, simply return it. + * Otherwise, it make a second call (with the new method) by passing in a + * storage type. + * + * This is for better performance reason. Put in short, the key note is that + * the old method is faster but may take several runs, while the new method + * is somewhat slower, and always succeed in one trial. + * See HDFS-11535 for more detail. + * + * @param scope range of nodes from which a node will be chosen + * @param excludedNodes nodes to be excluded from + * @param type the storage type we search for + * @return the chosen node + */ + public Node chooseRandomWithStorageTypeTwoTrial(final String scope, + final Collection excludedNodes, StorageType type) { + netlock.readLock().lock(); + try { + String searchScope; + String excludedScope; + if (scope.startsWith("~")) { + searchScope = NodeBase.ROOT; + excludedScope = scope.substring(1); + } else { + searchScope = scope; + excludedScope = null; + } + // next do a two-trial search + // first trial, call the old method, inherited from NetworkTopology + Node n = chooseRandom(searchScope, excludedScope, excludedNodes); + if (n == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("No node to choose."); + } + // this means there is simply no node to choose from + return null; + } + Preconditions.checkArgument(n instanceof DatanodeDescriptor); + DatanodeDescriptor dnDescriptor = (DatanodeDescriptor)n; + + if (dnDescriptor.hasStorageType(type)) { + // the first trial succeeded, just return + return dnDescriptor; + } else { + // otherwise, make the second trial by calling the new method + LOG.debug("First trial failed, node has no type {}, " + + "making second trial carrying this type", type); + return chooseRandomWithStorageType(searchScope, excludedScope, + excludedNodes, type); + } + } finally { + netlock.readLock().unlock(); + } + } + /** * Choose a random node based on given scope, excludedScope and excludedNodes * set. Although in general the topology has at most three layers, this class @@ -99,13 +162,10 @@ public class DFSNetworkTopology extends NetworkTopology { * all it's ancestors' storage counters accordingly, this way the excluded * root is out of the picture. * - * TODO : this function has duplicate code as NetworkTopology, need to - * refactor in the future. - * - * @param scope - * @param excludedScope - * @param excludedNodes - * @return + * @param scope the scope where we look for node. + * @param excludedScope the scope where the node must NOT be from. + * @param excludedNodes the returned node must not be in this set + * @return a node with required storage type */ @VisibleForTesting Node chooseRandomWithStorageType(final String scope, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/net/TestDFSNetworkTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/net/TestDFSNetworkTopology.java index 30ef2ac7f27..26d96b2309e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/net/TestDFSNetworkTopology.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/net/TestDFSNetworkTopology.java @@ -508,4 +508,61 @@ public class TestDFSNetworkTopology { assertEquals(1, innerl2d3r3.getSubtreeStorageCount(StorageType.DISK)); } + + @Test + public void testChooseRandomWithStorageTypeTwoTrial() throws Exception { + Node n; + DatanodeDescriptor dd; + n = CLUSTER.chooseRandomWithStorageType("/l2/d3/r4", null, null, + StorageType.ARCHIVE); + HashSet excluded = new HashSet<>(); + // exclude the host on r4 (since there is only one host, no randomness here) + excluded.add(n); + + // search with given scope being desired scope + for (int i = 0; i<10; i++) { + n = CLUSTER.chooseRandomWithStorageTypeTwoTrial( + "/l2/d3", null, StorageType.ARCHIVE); + assertTrue(n instanceof DatanodeDescriptor); + dd = (DatanodeDescriptor) n; + assertTrue(dd.getHostName().equals("host12") || + dd.getHostName().equals("host13")); + } + + for (int i = 0; i<10; i++) { + n = CLUSTER.chooseRandomWithStorageTypeTwoTrial( + "/l2/d3", excluded, StorageType.ARCHIVE); + assertTrue(n instanceof DatanodeDescriptor); + dd = (DatanodeDescriptor) n; + assertTrue(dd.getHostName().equals("host13")); + } + + // search with given scope being exclude scope + + // a total of 4 ramdisk nodes: + // /l1/d2/r3/host7, /l2/d3/r2/host10, /l2/d4/r1/host7 and /l2/d4/r1/host10 + // so if we exclude /l2/d4/r1, if should be always either host7 or host10 + for (int i = 0; i<10; i++) { + n = CLUSTER.chooseRandomWithStorageTypeTwoTrial( + "~/l2/d4", null, StorageType.RAM_DISK); + assertTrue(n instanceof DatanodeDescriptor); + dd = (DatanodeDescriptor) n; + assertTrue(dd.getHostName().equals("host7") || + dd.getHostName().equals("host10")); + } + + // similar to above, except that we also exclude host10 here. so it should + // always be host7 + n = CLUSTER.chooseRandomWithStorageType("/l2/d3/r2", null, null, + StorageType.RAM_DISK); + // add host10 to exclude + excluded.add(n); + for (int i = 0; i<10; i++) { + n = CLUSTER.chooseRandomWithStorageTypeTwoTrial( + "~/l2/d4", excluded, StorageType.RAM_DISK); + assertTrue(n instanceof DatanodeDescriptor); + dd = (DatanodeDescriptor) n; + assertTrue(dd.getHostName().equals("host7")); + } + } } From 01aca54a22c8586d232a8f79fe9977aeb8d09b83 Mon Sep 17 00:00:00 2001 From: Varun Saxena Date: Wed, 29 Mar 2017 01:53:20 +0530 Subject: [PATCH 149/188] YARN-5368. Memory leak in timeline server (Jonathan Eagles via Varun Saxena) --- .../timeline/RollingLevelDBTimelineStore.java | 263 ++++++++---------- 1 file changed, 117 insertions(+), 146 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/RollingLevelDBTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/RollingLevelDBTimelineStore.java index 4d38008c993..20e0379ac62 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/RollingLevelDBTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/RollingLevelDBTimelineStore.java @@ -275,9 +275,7 @@ public class RollingLevelDBTimelineStore extends AbstractService implements Path domainDBPath = new Path(dbPath, DOMAIN); Path starttimeDBPath = new Path(dbPath, STARTTIME); Path ownerDBPath = new Path(dbPath, OWNER); - FileSystem localFS = null; - try { - localFS = FileSystem.getLocal(conf); + try (FileSystem localFS = FileSystem.getLocal(conf)) { if (!localFS.exists(dbPath)) { if (!localFS.mkdirs(dbPath)) { throw new IOException("Couldn't create directory for leveldb " @@ -306,8 +304,6 @@ public class RollingLevelDBTimelineStore extends AbstractService implements } localFS.setPermission(ownerDBPath, LEVELDB_DIR_UMASK); } - } finally { - IOUtils.cleanup(LOG, localFS); } options.maxOpenFiles(conf.getInt( TIMELINE_SERVICE_LEVELDB_MAX_OPEN_FILES, @@ -408,19 +404,15 @@ public class RollingLevelDBTimelineStore extends AbstractService implements .add(writeReverseOrderedLong(revStartTime)).add(entityId) .getBytesForLookup(); - DBIterator iterator = null; - try { - DB db = entitydb.getDBForStartTime(revStartTime); - if (db == null) { - return null; - } - iterator = db.iterator(); + DB db = entitydb.getDBForStartTime(revStartTime); + if (db == null) { + return null; + } + try (DBIterator iterator = db.iterator()) { iterator.seek(prefix); return getEntity(entityId, entityType, revStartTime, fields, iterator, prefix, prefix.length); - } finally { - IOUtils.cleanup(LOG, iterator); } } @@ -533,62 +525,61 @@ public class RollingLevelDBTimelineStore extends AbstractService implements o2.length); } }); - DBIterator iterator = null; - try { + // look up start times for the specified entities // skip entities with no start time - for (String entityId : entityIds) { - byte[] startTime = getStartTime(entityId, entityType); - if (startTime != null) { - List entities = startTimeMap.get(startTime); - if (entities == null) { - entities = new ArrayList(); - startTimeMap.put(startTime, entities); - } - entities.add(new EntityIdentifier(entityId, entityType)); + for (String entityId : entityIds) { + byte[] startTime = getStartTime(entityId, entityType); + if (startTime != null) { + List entities = startTimeMap.get(startTime); + if (entities == null) { + entities = new ArrayList(); + startTimeMap.put(startTime, entities); } + entities.add(new EntityIdentifier(entityId, entityType)); } - for (Entry> entry : startTimeMap + } + for (Entry> entry : startTimeMap .entrySet()) { - // look up the events matching the given parameters (limit, - // start time, end time, event types) for entities whose start times - // were found and add the entities to the return list - byte[] revStartTime = entry.getKey(); - for (EntityIdentifier entityIdentifier : entry.getValue()) { - EventsOfOneEntity entity = new EventsOfOneEntity(); - entity.setEntityId(entityIdentifier.getId()); - entity.setEntityType(entityType); - events.addEvent(entity); - KeyBuilder kb = KeyBuilder.newInstance().add(entityType) - .add(revStartTime).add(entityIdentifier.getId()) - .add(EVENTS_COLUMN); - byte[] prefix = kb.getBytesForLookup(); - if (windowEnd == null) { - windowEnd = Long.MAX_VALUE; - } - byte[] revts = writeReverseOrderedLong(windowEnd); - kb.add(revts); - byte[] first = kb.getBytesForLookup(); - byte[] last = null; - if (windowStart != null) { - last = KeyBuilder.newInstance().add(prefix) - .add(writeReverseOrderedLong(windowStart)).getBytesForLookup(); - } - if (limit == null) { - limit = DEFAULT_LIMIT; - } - DB db = entitydb.getDBForStartTime(readReverseOrderedLong( - revStartTime, 0)); - if (db == null) { - continue; - } - iterator = db.iterator(); + // look up the events matching the given parameters (limit, + // start time, end time, event types) for entities whose start times + // were found and add the entities to the return list + byte[] revStartTime = entry.getKey(); + for (EntityIdentifier entityIdentifier : entry.getValue()) { + EventsOfOneEntity entity = new EventsOfOneEntity(); + entity.setEntityId(entityIdentifier.getId()); + entity.setEntityType(entityType); + events.addEvent(entity); + KeyBuilder kb = KeyBuilder.newInstance().add(entityType) + .add(revStartTime).add(entityIdentifier.getId()) + .add(EVENTS_COLUMN); + byte[] prefix = kb.getBytesForLookup(); + if (windowEnd == null) { + windowEnd = Long.MAX_VALUE; + } + byte[] revts = writeReverseOrderedLong(windowEnd); + kb.add(revts); + byte[] first = kb.getBytesForLookup(); + byte[] last = null; + if (windowStart != null) { + last = KeyBuilder.newInstance().add(prefix) + .add(writeReverseOrderedLong(windowStart)).getBytesForLookup(); + } + if (limit == null) { + limit = DEFAULT_LIMIT; + } + DB db = entitydb.getDBForStartTime(readReverseOrderedLong( + revStartTime, 0)); + if (db == null) { + continue; + } + try (DBIterator iterator = db.iterator()) { for (iterator.seek(first); entity.getEvents().size() < limit && iterator.hasNext(); iterator.next()) { byte[] key = iterator.peekNext().getKey(); if (!prefixMatches(prefix, prefix.length, key) || (last != null && WritableComparator.compareBytes(key, 0, - key.length, last, 0, last.length) > 0)) { + key.length, last, 0, last.length) > 0)) { break; } TimelineEvent event = getEntityEvent(eventType, key, prefix.length, @@ -599,8 +590,6 @@ public class RollingLevelDBTimelineStore extends AbstractService implements } } } - } finally { - IOUtils.cleanup(LOG, iterator); } return events; } @@ -657,66 +646,64 @@ public class RollingLevelDBTimelineStore extends AbstractService implements Long limit, Long starttime, Long endtime, String fromId, Long fromTs, Collection secondaryFilters, EnumSet fields, CheckAcl checkAcl, boolean usingPrimaryFilter) throws IOException { - DBIterator iterator = null; - try { - KeyBuilder kb = KeyBuilder.newInstance().add(base).add(entityType); - // only db keys matching the prefix (base + entity type) will be parsed - byte[] prefix = kb.getBytesForLookup(); - if (endtime == null) { - // if end time is null, place no restriction on end time - endtime = Long.MAX_VALUE; - } + KeyBuilder kb = KeyBuilder.newInstance().add(base).add(entityType); + // only db keys matching the prefix (base + entity type) will be parsed + byte[] prefix = kb.getBytesForLookup(); + if (endtime == null) { + // if end time is null, place no restriction on end time + endtime = Long.MAX_VALUE; + } - // Sanitize the fields parameter - if (fields == null) { - fields = EnumSet.allOf(Field.class); - } + // Sanitize the fields parameter + if (fields == null) { + fields = EnumSet.allOf(Field.class); + } - // construct a first key that will be seeked to using end time or fromId - long firstStartTime = Long.MAX_VALUE; - byte[] first = null; - if (fromId != null) { - Long fromIdStartTime = getStartTimeLong(fromId, entityType); - if (fromIdStartTime == null) { - // no start time for provided id, so return empty entities - return new TimelineEntities(); - } - if (fromIdStartTime <= endtime) { - // if provided id's start time falls before the end of the window, - // use it to construct the seek key - firstStartTime = fromIdStartTime; - first = kb.add(writeReverseOrderedLong(fromIdStartTime)).add(fromId) - .getBytesForLookup(); - } + // construct a first key that will be seeked to using end time or fromId + long firstStartTime = Long.MAX_VALUE; + byte[] first = null; + if (fromId != null) { + Long fromIdStartTime = getStartTimeLong(fromId, entityType); + if (fromIdStartTime == null) { + // no start time for provided id, so return empty entities + return new TimelineEntities(); } - // if seek key wasn't constructed using fromId, construct it using end ts - if (first == null) { - firstStartTime = endtime; - first = kb.add(writeReverseOrderedLong(endtime)).getBytesForLookup(); - } - byte[] last = null; - if (starttime != null) { - // if start time is not null, set a last key that will not be - // iterated past - last = KeyBuilder.newInstance().add(base).add(entityType) - .add(writeReverseOrderedLong(starttime)).getBytesForLookup(); - } - if (limit == null) { - // if limit is not specified, use the default - limit = DEFAULT_LIMIT; + if (fromIdStartTime <= endtime) { + // if provided id's start time falls before the end of the window, + // use it to construct the seek key + firstStartTime = fromIdStartTime; + first = kb.add(writeReverseOrderedLong(fromIdStartTime)).add(fromId) + .getBytesForLookup(); } + } + // if seek key wasn't constructed using fromId, construct it using end ts + if (first == null) { + firstStartTime = endtime; + first = kb.add(writeReverseOrderedLong(endtime)).getBytesForLookup(); + } + byte[] last = null; + if (starttime != null) { + // if start time is not null, set a last key that will not be + // iterated past + last = KeyBuilder.newInstance().add(base).add(entityType) + .add(writeReverseOrderedLong(starttime)).getBytesForLookup(); + } + if (limit == null) { + // if limit is not specified, use the default + limit = DEFAULT_LIMIT; + } - TimelineEntities entities = new TimelineEntities(); - RollingLevelDB rollingdb = null; - if (usingPrimaryFilter) { - rollingdb = indexdb; - } else { - rollingdb = entitydb; - } + TimelineEntities entities = new TimelineEntities(); + RollingLevelDB rollingdb = null; + if (usingPrimaryFilter) { + rollingdb = indexdb; + } else { + rollingdb = entitydb; + } - DB db = rollingdb.getDBForStartTime(firstStartTime); - while (entities.getEntities().size() < limit && db != null) { - iterator = db.iterator(); + DB db = rollingdb.getDBForStartTime(firstStartTime); + while (entities.getEntities().size() < limit && db != null) { + try (DBIterator iterator = db.iterator()) { iterator.seek(first); // iterate until one of the following conditions is met: limit is @@ -726,7 +713,7 @@ public class RollingLevelDBTimelineStore extends AbstractService implements byte[] key = iterator.peekNext().getKey(); if (!prefixMatches(prefix, prefix.length, key) || (last != null && WritableComparator.compareBytes(key, 0, - key.length, last, 0, last.length) > 0)) { + key.length, last, 0, last.length) > 0)) { break; } // read the start time and entity id from the current key @@ -814,10 +801,8 @@ public class RollingLevelDBTimelineStore extends AbstractService implements } db = rollingdb.getPreviousDB(db); } - return entities; - } finally { - IOUtils.cleanup(LOG, iterator); } + return entities; } /** @@ -1459,15 +1444,14 @@ public class RollingLevelDBTimelineStore extends AbstractService implements long startTimesCount = 0; WriteBatch writeBatch = null; - DBIterator iterator = null; - try { - writeBatch = starttimedb.createWriteBatch(); - ReadOptions readOptions = new ReadOptions(); - readOptions.fillCache(false); - iterator = starttimedb.iterator(readOptions); + ReadOptions readOptions = new ReadOptions(); + readOptions.fillCache(false); + try (DBIterator iterator = starttimedb.iterator(readOptions)) { + // seek to the first start time entry iterator.seekToFirst(); + writeBatch = starttimedb.createWriteBatch(); // evaluate each start time entry to see if it needs to be evicted or not while (iterator.hasNext()) { @@ -1513,7 +1497,6 @@ public class RollingLevelDBTimelineStore extends AbstractService implements + " start time entities earlier than " + minStartTime); } finally { IOUtils.cleanup(LOG, writeBatch); - IOUtils.cleanup(LOG, iterator); } return startTimesCount; } @@ -1598,11 +1581,9 @@ public class RollingLevelDBTimelineStore extends AbstractService implements // TODO: make data retention work with the domain data as well @Override public void put(TimelineDomain domain) throws IOException { - WriteBatch domainWriteBatch = null; - WriteBatch ownerWriteBatch = null; - try { - domainWriteBatch = domaindb.createWriteBatch(); - ownerWriteBatch = ownerdb.createWriteBatch(); + try (WriteBatch domainWriteBatch = domaindb.createWriteBatch(); + WriteBatch ownerWriteBatch = ownerdb.createWriteBatch();) { + if (domain.getId() == null || domain.getId().length() == 0) { throw new IllegalArgumentException("Domain doesn't have an ID"); } @@ -1682,9 +1663,6 @@ public class RollingLevelDBTimelineStore extends AbstractService implements ownerWriteBatch.put(ownerLookupEntryKey, timestamps); domaindb.write(domainWriteBatch); ownerdb.write(ownerWriteBatch); - } finally { - IOUtils.cleanup(LOG, domainWriteBatch); - IOUtils.cleanup(LOG, ownerWriteBatch); } } @@ -1709,26 +1687,21 @@ public class RollingLevelDBTimelineStore extends AbstractService implements @Override public TimelineDomain getDomain(String domainId) throws IOException { - DBIterator iterator = null; - try { + try (DBIterator iterator = domaindb.iterator()) { byte[] prefix = KeyBuilder.newInstance().add(domainId) .getBytesForLookup(); - iterator = domaindb.iterator(); iterator.seek(prefix); return getTimelineDomain(iterator, domainId, prefix); - } finally { - IOUtils.cleanup(LOG, iterator); } } @Override public TimelineDomains getDomains(String owner) throws IOException { - DBIterator iterator = null; - try { + try (DBIterator iterator = ownerdb.iterator()) { byte[] prefix = KeyBuilder.newInstance().add(owner).getBytesForLookup(); + iterator.seek(prefix); List domains = new ArrayList(); - for (iterator = ownerdb.iterator(), iterator.seek(prefix); iterator - .hasNext();) { + while (iterator.hasNext()) { byte[] key = iterator.peekNext().getKey(); if (!prefixMatches(prefix, prefix.length, key)) { break; @@ -1761,8 +1734,6 @@ public class RollingLevelDBTimelineStore extends AbstractService implements TimelineDomains domainsToReturn = new TimelineDomains(); domainsToReturn.addDomains(domains); return domainsToReturn; - } finally { - IOUtils.cleanup(LOG, iterator); } } From 063b513b1c10987461caab3d26c8543c6e657bf7 Mon Sep 17 00:00:00 2001 From: Varun Saxena Date: Wed, 29 Mar 2017 03:48:03 +0530 Subject: [PATCH 150/188] YARN-6357. Implement putEntitiesAsync API in TimelineCollector (Haibo Chen via Varun Saxena) --- .../collector/TimelineCollector.java | 31 +++++++-- .../TimelineCollectorWebService.java | 12 ++-- .../collector/TestTimelineCollector.java | 63 +++++++++++++++++++ 3 files changed, 96 insertions(+), 10 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java index 2fc30333133..353066bd775 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java @@ -133,19 +133,35 @@ public abstract class TimelineCollector extends CompositeService { public TimelineWriteResponse putEntities(TimelineEntities entities, UserGroupInformation callerUgi) throws IOException { if (LOG.isDebugEnabled()) { - LOG.debug("SUCCESS - TIMELINE V2 PROTOTYPE"); LOG.debug("putEntities(entities=" + entities + ", callerUgi=" + callerUgi + ")"); } - TimelineCollectorContext context = getTimelineEntityContext(); + TimelineWriteResponse response = writeTimelineEntities(entities); + flushBufferedTimelineEntities(); + + return response; + } + + private TimelineWriteResponse writeTimelineEntities( + TimelineEntities entities) throws IOException { // Update application metrics for aggregation updateAggregateStatus(entities, aggregationGroups, getEntityTypesSkipAggregation()); + final TimelineCollectorContext context = getTimelineEntityContext(); return writer.write(context.getClusterId(), context.getUserId(), - context.getFlowName(), context.getFlowVersion(), context.getFlowRunId(), - context.getAppId(), entities); + context.getFlowName(), context.getFlowVersion(), + context.getFlowRunId(), context.getAppId(), entities); + } + + /** + * Flush buffered timeline entities, if any. + * @throws IOException if there is any exception encountered while + * flushing buffered entities. + */ + private void flushBufferedTimelineEntities() throws IOException { + writer.flush(); } /** @@ -158,14 +174,17 @@ public abstract class TimelineCollector extends CompositeService { * * @param entities entities to post * @param callerUgi the caller UGI + * @throws IOException if there is any exception encounted while putting + * entities. */ public void putEntitiesAsync(TimelineEntities entities, - UserGroupInformation callerUgi) { - // TODO implement + UserGroupInformation callerUgi) throws IOException { if (LOG.isDebugEnabled()) { LOG.debug("putEntitiesAsync(entities=" + entities + ", callerUgi=" + callerUgi + ")"); } + + writeTimelineEntities(entities); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java index f36c63609c2..fe04b7afc5b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java @@ -152,9 +152,6 @@ public class TimelineCollectorWebService { throw new ForbiddenException(msg); } - // TODO how to express async posts and handle them - boolean isAsync = async != null && async.trim().equalsIgnoreCase("true"); - try { ApplicationId appID = parseApplicationId(appId); if (appID == null) { @@ -169,7 +166,14 @@ public class TimelineCollectorWebService { throw new NotFoundException(); // different exception? } - collector.putEntities(processTimelineEntities(entities), callerUgi); + boolean isAsync = async != null && async.trim().equalsIgnoreCase("true"); + if (isAsync) { + collector.putEntitiesAsync( + processTimelineEntities(entities), callerUgi); + } else { + collector.putEntities(processTimelineEntities(entities), callerUgi); + } + return Response.ok().build(); } catch (Exception e) { LOG.error("Error putting entities", e); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollector.java index 5b4dc50deed..a55f2276a96 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollector.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollector.java @@ -18,17 +18,27 @@ package org.apache.hadoop.yarn.server.timelineservice.collector; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetricOperation; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric; +import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter; import org.junit.Test; +import java.io.IOException; import java.util.HashSet; import java.util.Set; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; public class TestTimelineCollector { @@ -124,4 +134,57 @@ public class TestTimelineCollector { } } + + /** + * Test TimelineCollector's interaction with TimelineWriter upon + * putEntity() calls. + */ + @Test + public void testPutEntity() throws IOException { + TimelineWriter writer = mock(TimelineWriter.class); + TimelineCollector collector = new TimelineCollectorForTest(writer); + + TimelineEntities entities = generateTestEntities(1, 1); + collector.putEntities( + entities, UserGroupInformation.createRemoteUser("test-user")); + + verify(writer, times(1)).write( + anyString(), anyString(), anyString(), anyString(), anyLong(), + anyString(), any(TimelineEntities.class)); + verify(writer, times(1)).flush(); + } + + /** + * Test TimelineCollector's interaction with TimelineWriter upon + * putEntityAsync() calls. + */ + @Test + public void testPutEntityAsync() throws IOException { + TimelineWriter writer = mock(TimelineWriter.class); + TimelineCollector collector = new TimelineCollectorForTest(writer); + + TimelineEntities entities = generateTestEntities(1, 1); + collector.putEntitiesAsync( + entities, UserGroupInformation.createRemoteUser("test-user")); + + verify(writer, times(1)).write( + anyString(), anyString(), anyString(), anyString(), anyLong(), + anyString(), any(TimelineEntities.class)); + verify(writer, never()).flush(); + } + + private static class TimelineCollectorForTest extends TimelineCollector { + private final TimelineCollectorContext context = + new TimelineCollectorContext(); + + TimelineCollectorForTest(TimelineWriter writer) { + super("TimelineCollectorForTest"); + setWriter(writer); + } + + @Override + public TimelineCollectorContext getTimelineEntityContext() { + return context; + } + } } From 82fb9ce8df7a534a4cbcec624d7b6c0b33e79123 Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Wed, 29 Mar 2017 10:35:30 +0900 Subject: [PATCH 151/188] YARN-6329. Remove unnecessary TODO comment from AppLogAggregatorImpl.java. Contributed by victor bertschinger. --- .../containermanager/logaggregation/AppLogAggregatorImpl.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java index 065854e98df..d70acc940e4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java @@ -573,8 +573,6 @@ public class AppLogAggregatorImpl implements AppLogAggregator { (remoteNodeLogFileForApp.getName() + LogAggregationUtils.TMP_FILE_SUFFIX)); } - // TODO: The condition: containerId.getId() == 1 to determine an AM container - // is not always true. private boolean shouldUploadLogs(ContainerLogContext logContext) { return logAggPolicy.shouldDoLogAggregation(logContext); } From 0e6f8e4bc6642f90dc7b33848bfb1129ec20ee49 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Tue, 28 Mar 2017 22:14:03 -0700 Subject: [PATCH 152/188] HDFS-10971. Distcp should not copy replication factor if source file is erasure coded. Contributed by Manoj Govindassamy. --- .../hadoop/tools/CopyListingFileStatus.java | 4 + .../mapred/RetriableFileCopyCommand.java | 3 + .../apache/hadoop/tools/util/DistCpUtils.java | 9 +- .../tools/TestCopyListingFileStatus.java | 1 + .../hadoop/tools/util/TestDistCpUtils.java | 123 +++++++++++++++++- 5 files changed, 137 insertions(+), 3 deletions(-) diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java index 2b1e7e4ce47..00d4b325053 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java @@ -159,6 +159,10 @@ public final class CopyListingFileStatus implements Writable { return permission; } + public boolean isErasureCoded() { + return getPermission().getErasureCodedBit(); + } + /** * Returns the full logical ACL. * diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java index d1cdfdd5485..06acd78a8a1 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java @@ -167,6 +167,9 @@ public class RetriableFileCopyCommand extends RetriableCommand { FsPermission.getUMask(targetFS.getConf())); final OutputStream outStream; if (action == FileAction.OVERWRITE) { + // If there is an erasure coding policy set on the target directory, + // files will be written to the target directory using the same EC policy. + // The replication factor of the source file is ignored and not preserved. final short repl = getReplicationFactor(fileAttributes, source, targetFS, targetPath); final long blockSize = getBlockSize(fileAttributes, source, diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java index c308e6f1f90..76bc4c56268 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java @@ -236,8 +236,13 @@ public class DistCpUtils { } } - if (attributes.contains(FileAttribute.REPLICATION) && !targetFileStatus.isDirectory() && - (srcFileStatus.getReplication() != targetFileStatus.getReplication())) { + // The replication factor can only be preserved for replicated files. + // It is ignored when either the source or target file are erasure coded. + if (attributes.contains(FileAttribute.REPLICATION) && + !targetFileStatus.isDirectory() && + !targetFileStatus.isErasureCoded() && + !srcFileStatus.isErasureCoded() && + srcFileStatus.getReplication() != targetFileStatus.getReplication()) { targetFS.setReplication(path, srcFileStatus.getReplication()); } diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListingFileStatus.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListingFileStatus.java index f512ef6d9d8..8efc5cf9942 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListingFileStatus.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListingFileStatus.java @@ -62,6 +62,7 @@ public class TestCopyListingFileStatus { assertEquals(stat.getOwner(), clfs.getOwner()); assertEquals(stat.getGroup(), clfs.getGroup()); assertEquals(stat.getPath(), clfs.getPath()); + assertEquals(stat.isErasureCoded(), clfs.isErasureCoded()); } } diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/TestDistCpUtils.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/TestDistCpUtils.java index 8c79becfa0b..c42e5465680 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/TestDistCpUtils.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/TestDistCpUtils.java @@ -18,6 +18,10 @@ package org.apache.hadoop.tools.util; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; + import java.io.IOException; import java.io.OutputStream; import java.util.EnumSet; @@ -31,11 +35,15 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.server.namenode.INodeFile; +import org.apache.hadoop.hdfs.tools.ECAdmin; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.tools.CopyListingFileStatus; import org.apache.hadoop.tools.DistCpOptionSwitch; import org.apache.hadoop.tools.DistCpOptions.FileAttribute; +import org.apache.hadoop.util.ToolRunner; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -52,8 +60,10 @@ public class TestDistCpUtils { @BeforeClass public static void create() throws IOException { + config.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + "XOR-2-1-64k"); cluster = new MiniDFSCluster.Builder(config) - .numDataNodes(1) + .numDataNodes(2) .format(true) .build(); } @@ -539,6 +549,117 @@ public class TestDistCpUtils { Assert.assertTrue(srcStatus.getReplication() == dstStatus.getReplication()); } + @Test (timeout = 60000) + public void testReplFactorNotPreservedOnErasureCodedFile() throws Exception { + FileSystem fs = FileSystem.get(config); + + // Case 1: Verify replication attribute not preserved when the source + // file is erasure coded and the target file is replicated. + Path srcECDir = new Path("/tmp/srcECDir"); + Path srcECFile = new Path(srcECDir, "srcECFile"); + Path dstReplDir = new Path("/tmp/dstReplDir"); + Path dstReplFile = new Path(dstReplDir, "destReplFile"); + fs.mkdirs(srcECDir); + fs.mkdirs(dstReplDir); + String[] args = {"-setPolicy", "-path", "/tmp/srcECDir", + "-policy", "XOR-2-1-64k"}; + int res = ToolRunner.run(config, new ECAdmin(config), args); + assertEquals("Setting EC policy should succeed!", 0, res); + verifyReplFactorNotPreservedOnErasureCodedFile(srcECFile, true, + dstReplFile, false); + + // Case 2: Verify replication attribute not preserved when the source + // file is replicated and the target file is erasure coded. + Path srcReplDir = new Path("/tmp/srcReplDir"); + Path srcReplFile = new Path(srcReplDir, "srcReplFile"); + Path dstECDir = new Path("/tmp/dstECDir"); + Path dstECFile = new Path(dstECDir, "destECFile"); + fs.mkdirs(srcReplDir); + fs.mkdirs(dstECDir); + args = new String[]{"-setPolicy", "-path", "/tmp/dstECDir", + "-policy", "XOR-2-1-64k"}; + res = ToolRunner.run(config, new ECAdmin(config), args); + assertEquals("Setting EC policy should succeed!", 0, res); + verifyReplFactorNotPreservedOnErasureCodedFile(srcReplFile, + false, dstECFile, true); + + // Case 3: Verify replication attribute not altered from the default + // INodeFile.DEFAULT_REPL_FOR_STRIPED_BLOCKS when both source and + // target files are erasure coded. + verifyReplFactorNotPreservedOnErasureCodedFile(srcECFile, + true, dstECFile, true); + } + + private void verifyReplFactorNotPreservedOnErasureCodedFile(Path srcFile, + boolean isSrcEC, Path dstFile, boolean isDstEC) throws Exception { + FileSystem fs = FileSystem.get(config); + createFile(fs, srcFile); + CopyListingFileStatus srcStatus = new CopyListingFileStatus( + fs.getFileStatus(srcFile)); + if (isSrcEC) { + assertTrue(srcFile + "should be erasure coded!", + srcStatus.isErasureCoded()); + assertEquals(INodeFile.DEFAULT_REPL_FOR_STRIPED_BLOCKS, + srcStatus.getReplication()); + } else { + assertEquals("Unexpected replication factor for " + srcFile, + fs.getDefaultReplication(srcFile), srcStatus.getReplication()); + } + + createFile(fs, dstFile); + CopyListingFileStatus dstStatus = new CopyListingFileStatus( + fs.getFileStatus(dstFile)); + if (isDstEC) { + assertTrue(dstFile + "should be erasure coded!", + dstStatus.isErasureCoded()); + assertEquals("Unexpected replication factor for erasure coded file!", + INodeFile.DEFAULT_REPL_FOR_STRIPED_BLOCKS, + dstStatus.getReplication()); + } else { + assertEquals("Unexpected replication factor for " + dstFile, + fs.getDefaultReplication(dstFile), dstStatus.getReplication()); + } + + // Let srcFile and dstFile differ on their FileAttribute + fs.setPermission(srcFile, fullPerm); + fs.setOwner(srcFile, "ec", "ec-group"); + fs.setTimes(srcFile, 0, 0); + + fs.setPermission(dstFile, noPerm); + fs.setOwner(dstFile, "normal", "normal-group"); + fs.setTimes(dstFile, 100, 100); + + // Running preserve operations only for replication attribute + srcStatus = new CopyListingFileStatus(fs.getFileStatus(srcFile)); + EnumSet attributes = EnumSet.of(FileAttribute.REPLICATION); + DistCpUtils.preserve(fs, dstFile, srcStatus, attributes, false); + dstStatus = new CopyListingFileStatus(fs.getFileStatus(dstFile)); + + assertFalse("Permission for " + srcFile + " and " + dstFile + + " should not be same after preserve only for replication attr!", + srcStatus.getPermission().equals(dstStatus.getPermission())); + assertFalse("File ownership should not match!", + srcStatus.getOwner().equals(dstStatus.getOwner())); + assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup())); + assertFalse(srcStatus.getAccessTime() == dstStatus.getAccessTime()); + assertFalse( + srcStatus.getModificationTime() == dstStatus.getModificationTime()); + if (isDstEC) { + assertEquals("Unexpected replication factor for erasure coded file!", + INodeFile.DEFAULT_REPL_FOR_STRIPED_BLOCKS, + dstStatus.getReplication()); + } else { + assertEquals(dstFile + " replication factor should be same as dst " + + "filesystem!", fs.getDefaultReplication(dstFile), + dstStatus.getReplication()); + } + if (!isSrcEC || !isDstEC) { + assertFalse(dstFile + " replication factor should not be " + + "same as " + srcFile, + srcStatus.getReplication() == dstStatus.getReplication()); + } + } + @Test public void testPreserveTimestampOnFile() throws IOException { FileSystem fs = FileSystem.get(config); From 84d787b9d51196010495d51dc5ebf66c01c340ab Mon Sep 17 00:00:00 2001 From: Rakesh Radhakrishnan Date: Wed, 29 Mar 2017 11:41:48 +0530 Subject: [PATCH 153/188] HDFS-11541. Call RawErasureEncoder and RawErasureDecoder release() methods. Contributed by SammiChen. --- .../hadoop/hdfs/DFSStripedInputStream.java | 20 +++++---- .../hadoop/hdfs/DFSStripedOutputStream.java | 1 + .../StripedBlockChecksumReconstructor.java | 42 ++++++++++--------- .../StripedBlockReconstructor.java | 1 + .../erasurecode/StripedReconstructor.java | 6 +++ 5 files changed, 43 insertions(+), 27 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java index 922f74eaecc..07e660934db 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java @@ -177,14 +177,18 @@ public class DFSStripedInputStream extends DFSInputStream { @Override public synchronized void close() throws IOException { - super.close(); - if (curStripeBuf != null) { - BUFFER_POOL.putBuffer(curStripeBuf); - curStripeBuf = null; - } - if (parityBuf != null) { - BUFFER_POOL.putBuffer(parityBuf); - parityBuf = null; + try { + super.close(); + } finally { + if (curStripeBuf != null) { + BUFFER_POOL.putBuffer(curStripeBuf); + curStripeBuf = null; + } + if (parityBuf != null) { + BUFFER_POOL.putBuffer(parityBuf); + parityBuf = null; + } + decoder.release(); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java index 52fc5ebf71e..22b30e93063 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java @@ -1033,6 +1033,7 @@ public class DFSStripedOutputStream extends DFSOutputStream { setClosed(); // shutdown executor of flushAll tasks flushAllExecutor.shutdownNow(); + encoder.release(); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java index 95556189903..69173173fee 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockChecksumReconstructor.java @@ -75,29 +75,33 @@ public class StripedBlockChecksumReconstructor extends StripedReconstructor { public void reconstruct() throws IOException { MessageDigest digester = MD5Hash.getDigester(); long maxTargetLength = getMaxTargetLength(); - while (requestedLen > 0 && getPositionInBlock() < maxTargetLength) { - long remaining = maxTargetLength - getPositionInBlock(); - final int toReconstructLen = (int) Math - .min(getStripedReader().getBufferSize(), remaining); - // step1: read from minimum source DNs required for reconstruction. - // The returned success list is the source DNs we do real read from - getStripedReader().readMinimumSources(toReconstructLen); + try { + while (requestedLen > 0 && getPositionInBlock() < maxTargetLength) { + long remaining = maxTargetLength - getPositionInBlock(); + final int toReconstructLen = (int) Math + .min(getStripedReader().getBufferSize(), remaining); + // step1: read from minimum source DNs required for reconstruction. + // The returned success list is the source DNs we do real read from + getStripedReader().readMinimumSources(toReconstructLen); - // step2: decode to reconstruct targets - reconstructTargets(toReconstructLen); + // step2: decode to reconstruct targets + reconstructTargets(toReconstructLen); - // step3: calculate checksum - checksumDataLen += checksumWithTargetOutput(targetBuffer.array(), - toReconstructLen, digester); + // step3: calculate checksum + checksumDataLen += checksumWithTargetOutput(targetBuffer.array(), + toReconstructLen, digester); - updatePositionInBlock(toReconstructLen); - requestedLen -= toReconstructLen; - clearBuffers(); + updatePositionInBlock(toReconstructLen); + requestedLen -= toReconstructLen; + clearBuffers(); + } + + byte[] digest = digester.digest(); + md5 = new MD5Hash(digest); + md5.write(checksumWriter); + } finally { + cleanup(); } - - byte[] digest = digester.digest(); - md5 = new MD5Hash(digest); - md5.write(checksumWriter); } private long checksumWithTargetOutput(byte[] outputData, int toReconstructLen, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java index a1da536ba58..1119bbbd230 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java @@ -74,6 +74,7 @@ class StripedBlockReconstructor extends StripedReconstructor metrics.incrECReconstructionBytesWritten(getBytesWritten()); getStripedReader().close(); stripedWriter.close(); + cleanup(); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java index cd17864c943..b8433c7b6c3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java @@ -253,6 +253,12 @@ abstract class StripedReconstructor { return decoder; } + void cleanup() { + if (decoder != null) { + decoder.release(); + } + } + StripedReader getStripedReader() { return stripedReader; } From 523f467d939d80e2bc162e1f47be497109783061 Mon Sep 17 00:00:00 2001 From: Jonathan Eagles Date: Wed, 29 Mar 2017 10:12:02 -0500 Subject: [PATCH 154/188] HADOOP-14216. Improve Configuration XML Parsing Performance (jeagles) --- hadoop-common-project/hadoop-common/pom.xml | 10 + .../org/apache/hadoop/conf/Configuration.java | 311 +++++++++++------- .../apache/hadoop/conf/TestConfiguration.java | 62 +++- hadoop-project/pom.xml | 10 + 4 files changed, 263 insertions(+), 130 deletions(-) diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 961c9028393..ee82df00b44 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -313,6 +313,16 @@ com.fasterxml.jackson.core jackson-databind + + org.codehaus.woodstox + stax2-api + compile + + + com.fasterxml + aalto-xml + compile + diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index da4f6ac6b3f..36845e393bd 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -18,6 +18,7 @@ package org.apache.hadoop.conf; +import com.fasterxml.aalto.stax.InputFactoryImpl; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; import com.google.common.annotations.VisibleForTesting; @@ -65,9 +66,11 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; -import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; +import javax.xml.stream.XMLStreamConstants; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; import javax.xml.transform.Transformer; import javax.xml.transform.TransformerException; import javax.xml.transform.TransformerFactory; @@ -93,14 +96,10 @@ import org.apache.hadoop.security.alias.CredentialProviderFactory; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.StringInterner; import org.apache.hadoop.util.StringUtils; -import org.w3c.dom.Attr; -import org.w3c.dom.DOMException; +import org.codehaus.stax2.XMLInputFactory2; +import org.codehaus.stax2.XMLStreamReader2; import org.w3c.dom.Document; import org.w3c.dom.Element; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; -import org.w3c.dom.Text; -import org.xml.sax.SAXException; import com.google.common.base.Preconditions; import com.google.common.base.Strings; @@ -280,7 +279,13 @@ public class Configuration implements Iterable>, * the key most recently */ private Map updatingResource; - + + /** + * Specify exact input factory to avoid time finding correct one. + * Factory is reusable across un-synchronized threads once initialized + */ + private static final XMLInputFactory2 factory = new InputFactoryImpl(); + /** * Class to keep the information about the keys which replace the deprecated * ones. @@ -689,7 +694,7 @@ public class Configuration implements Iterable>, addDefaultResource("hadoop-site.xml"); } } - + private Properties properties; private Properties overlay; private ClassLoader classLoader; @@ -2613,8 +2618,8 @@ public class Configuration implements Iterable>, return configMap; } - private Document parse(DocumentBuilder builder, URL url) - throws IOException, SAXException { + private XMLStreamReader parse(URL url) + throws IOException, XMLStreamException { if (!quietmode) { if (LOG.isDebugEnabled()) { LOG.debug("parsing URL " + url); @@ -2630,23 +2635,18 @@ public class Configuration implements Iterable>, // with other users. connection.setUseCaches(false); } - return parse(builder, connection.getInputStream(), url.toString()); + return parse(connection.getInputStream(), url.toString()); } - private Document parse(DocumentBuilder builder, InputStream is, - String systemId) throws IOException, SAXException { + private XMLStreamReader parse(InputStream is, + String systemId) throws IOException, XMLStreamException { if (!quietmode) { LOG.debug("parsing input stream " + is); } if (is == null) { return null; } - try { - return (systemId == null) ? builder.parse(is) : builder.parse(is, - systemId); - } finally { - is.close(); - } + return factory.createXMLStreamReader(systemId, is); } private void loadResources(Properties properties, @@ -2666,37 +2666,20 @@ public class Configuration implements Iterable>, } } - private Resource loadResource(Properties properties, Resource wrapper, boolean quiet) { + private Resource loadResource(Properties properties, + Resource wrapper, boolean quiet) { String name = UNKNOWN_RESOURCE; try { Object resource = wrapper.getResource(); name = wrapper.getName(); - - DocumentBuilderFactory docBuilderFactory - = DocumentBuilderFactory.newInstance(); - //ignore all comments inside the xml file - docBuilderFactory.setIgnoringComments(true); - - //allow includes in the xml file - docBuilderFactory.setNamespaceAware(true); - try { - docBuilderFactory.setXIncludeAware(true); - } catch (UnsupportedOperationException e) { - LOG.error("Failed to set setXIncludeAware(true) for parser " - + docBuilderFactory - + ":" + e, - e); - } - DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); - Document doc = null; - Element root = null; + XMLStreamReader2 reader = null; boolean returnCachedProperties = false; - + if (resource instanceof URL) { // an URL resource - doc = parse(builder, (URL)resource); + reader = (XMLStreamReader2)parse((URL)resource); } else if (resource instanceof String) { // a CLASSPATH resource URL url = getResource((String)resource); - doc = parse(builder, url); + reader = (XMLStreamReader2)parse(url); } else if (resource instanceof Path) { // a file resource // Can't use FileSystem API or we get an infinite loop // since FileSystem uses Configuration API. Use java.io.File instead. @@ -2706,104 +2689,188 @@ public class Configuration implements Iterable>, if (!quiet) { LOG.debug("parsing File " + file); } - doc = parse(builder, new BufferedInputStream( + reader = (XMLStreamReader2)parse(new BufferedInputStream( new FileInputStream(file)), ((Path)resource).toString()); } } else if (resource instanceof InputStream) { - doc = parse(builder, (InputStream) resource, null); + reader = (XMLStreamReader2)parse((InputStream)resource, null); returnCachedProperties = true; } else if (resource instanceof Properties) { overlay(properties, (Properties)resource); - } else if (resource instanceof Element) { - root = (Element)resource; } - if (root == null) { - if (doc == null) { - if (quiet) { - return null; - } - throw new RuntimeException(resource + " not found"); + if (reader == null) { + if (quiet) { + return null; } - root = doc.getDocumentElement(); + throw new RuntimeException(resource + " not found"); } Properties toAddTo = properties; if(returnCachedProperties) { toAddTo = new Properties(); } - if (!"configuration".equals(root.getTagName())) - LOG.fatal("bad conf file: top-level element not "); - NodeList props = root.getChildNodes(); DeprecationContext deprecations = deprecationContext.get(); - for (int i = 0; i < props.getLength(); i++) { - Node propNode = props.item(i); - if (!(propNode instanceof Element)) - continue; - Element prop = (Element)propNode; - if ("configuration".equals(prop.getTagName())) { - loadResource(toAddTo, new Resource(prop, name), quiet); - continue; - } - if (!"property".equals(prop.getTagName())) - LOG.warn("bad conf file: element not "); - String attr = null; - String value = null; - boolean finalParameter = false; - LinkedList source = new LinkedList(); + StringBuilder token = new StringBuilder(); + String confName = null; + String confValue = null; + boolean confFinal = false; + boolean fallbackAllowed = false; + boolean fallbackEntered = false; + boolean parseToken = false; + LinkedList confSource = new LinkedList(); - Attr propAttr = prop.getAttributeNode("name"); - if (propAttr != null) - attr = StringInterner.weakIntern(propAttr.getValue()); - propAttr = prop.getAttributeNode("value"); - if (propAttr != null) - value = StringInterner.weakIntern(propAttr.getValue()); - propAttr = prop.getAttributeNode("final"); - if (propAttr != null) - finalParameter = "true".equals(propAttr.getValue()); - propAttr = prop.getAttributeNode("source"); - if (propAttr != null) - source.add(StringInterner.weakIntern(propAttr.getValue())); + while (reader.hasNext()) { + switch (reader.next()) { + case XMLStreamConstants.START_ELEMENT: + switch (reader.getLocalName()) { + case "property": + confName = null; + confValue = null; + confFinal = false; + confSource.clear(); - NodeList fields = prop.getChildNodes(); - for (int j = 0; j < fields.getLength(); j++) { - Node fieldNode = fields.item(j); - if (!(fieldNode instanceof Element)) - continue; - Element field = (Element)fieldNode; - if ("name".equals(field.getTagName()) && field.hasChildNodes()) - attr = StringInterner.weakIntern( - ((Text)field.getFirstChild()).getData().trim()); - if ("value".equals(field.getTagName()) && field.hasChildNodes()) - value = StringInterner.weakIntern( - ((Text)field.getFirstChild()).getData()); - if ("final".equals(field.getTagName()) && field.hasChildNodes()) - finalParameter = "true".equals(((Text)field.getFirstChild()).getData()); - if ("source".equals(field.getTagName()) && field.hasChildNodes()) - source.add(StringInterner.weakIntern( - ((Text)field.getFirstChild()).getData())); - } - source.add(name); - - // Ignore this parameter if it has already been marked as 'final' - if (attr != null) { - if (deprecations.getDeprecatedKeyMap().containsKey(attr)) { - DeprecatedKeyInfo keyInfo = - deprecations.getDeprecatedKeyMap().get(attr); - keyInfo.clearAccessed(); - for (String key:keyInfo.newKeys) { - // update new keys with deprecated key's value - loadProperty(toAddTo, name, key, value, finalParameter, - source.toArray(new String[source.size()])); + // First test for short format configuration + int attrCount = reader.getAttributeCount(); + for (int i = 0; i < attrCount; i++) { + String propertyAttr = reader.getAttributeLocalName(i); + if ("name".equals(propertyAttr)) { + confName = StringInterner.weakIntern( + reader.getAttributeValue(i)); + } else if ("value".equals(propertyAttr)) { + confValue = StringInterner.weakIntern( + reader.getAttributeValue(i)); + } else if ("final".equals(propertyAttr)) { + confFinal = "true".equals(reader.getAttributeValue(i)); + } else if ("source".equals(propertyAttr)) { + confSource.add(StringInterner.weakIntern( + reader.getAttributeValue(i))); + } } + break; + case "name": + case "value": + case "final": + case "source": + parseToken = true; + token.setLength(0); + break; + case "include": + if (!"xi".equals(reader.getPrefix())) { + break; + } + // Determine href for xi:include + String confInclude = null; + attrCount = reader.getAttributeCount(); + for (int i = 0; i < attrCount; i++) { + String attrName = reader.getAttributeLocalName(i); + if ("href".equals(attrName)) { + confInclude = reader.getAttributeValue(i); + } + } + if (confInclude == null) { + break; + } + // Determine if the included resource is a classpath resource + // otherwise fallback to a file resource + // xi:include are treated as inline and retain current source + URL include = getResource(confInclude); + if (include != null) { + Resource classpathResource = new Resource(include, name); + loadResource(properties, classpathResource, quiet); + } else { + File href = new File(confInclude); + if (!href.isAbsolute()) { + // Included resources are relative to the current resource + File baseFile = new File(name).getParentFile(); + href = new File(baseFile, href.getPath()); + } + if (!href.exists()) { + // Resource errors are non-fatal iff there is 1 xi:fallback + fallbackAllowed = true; + break; + } + Resource uriResource = new Resource(href.toURI().toURL(), name); + loadResource(properties, uriResource, quiet); + } + break; + case "fallback": + if (!"xi".equals(reader.getPrefix())) { + break; + } + fallbackEntered = true; + break; + case "configuration": + break; + default: + break; } - else { - loadProperty(toAddTo, name, attr, value, finalParameter, - source.toArray(new String[source.size()])); + break; + + case XMLStreamConstants.CHARACTERS: + if (parseToken) { + char[] text = reader.getTextCharacters(); + token.append(text, reader.getTextStart(), reader.getTextLength()); } + break; + + case XMLStreamConstants.END_ELEMENT: + switch (reader.getLocalName()) { + case "name": + if (token.length() > 0) { + confName = StringInterner.weakIntern(token.toString().trim()); + } + break; + case "value": + if (token.length() > 0) { + confValue = StringInterner.weakIntern(token.toString()); + } + break; + case "final": + confFinal = "true".equals(token.toString()); + break; + case "source": + confSource.add(StringInterner.weakIntern(token.toString())); + break; + case "include": + if (!"xi".equals(reader.getPrefix())) { + break; + } + if (fallbackAllowed && !fallbackEntered) { + throw new IOException("Fetch fail on include with no " + + "fallback while loading '" + name + "'"); + } + fallbackAllowed = false; + fallbackEntered = false; + break; + case "property": + if (confName == null || (!fallbackAllowed && fallbackEntered)) { + break; + } + confSource.add(name); + DeprecatedKeyInfo keyInfo = + deprecations.getDeprecatedKeyMap().get(confName); + if (keyInfo != null) { + keyInfo.clearAccessed(); + for (String key : keyInfo.newKeys) { + // update new keys with deprecated key's value + loadProperty(toAddTo, name, key, confValue, confFinal, + confSource.toArray(new String[confSource.size()])); + } + } else { + loadProperty(toAddTo, name, confName, confValue, confFinal, + confSource.toArray(new String[confSource.size()])); + } + break; + default: + break; + } + default: + break; } } - + reader.close(); + if (returnCachedProperties) { overlay(properties, toAddTo); return new Resource(toAddTo, name); @@ -2812,15 +2879,9 @@ public class Configuration implements Iterable>, } catch (IOException e) { LOG.fatal("error parsing conf " + name, e); throw new RuntimeException(e); - } catch (DOMException e) { + } catch (XMLStreamException e) { LOG.fatal("error parsing conf " + name, e); throw new RuntimeException(e); - } catch (SAXException e) { - LOG.fatal("error parsing conf " + name, e); - throw new RuntimeException(e); - } catch (ParserConfigurationException e) { - LOG.fatal("error parsing conf " + name , e); - throw new RuntimeException(e); } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java index 1f977865e22..6f4c26cf865 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java @@ -99,10 +99,22 @@ public class TestConfiguration extends TestCase { out.close(); } - private void addInclude(String filename) throws IOException{ - out.write("\n "); + private void startInclude(String filename) throws IOException { + out.write("\n "); } - + + private void endInclude() throws IOException{ + out.write("\n "); + } + + private void startFallback() throws IOException { + out.write("\n "); + } + + private void endFallback() throws IOException { + out.write("\n "); + } + public void testInputStreamResource() throws Exception { StringWriter writer = new StringWriter(); out = new BufferedWriter(writer); @@ -507,7 +519,8 @@ public class TestConfiguration extends TestCase { out=new BufferedWriter(new FileWriter(CONFIG)); startConfig(); - addInclude(CONFIG2); + startInclude(CONFIG2); + endInclude(); appendProperty("e","f"); appendProperty("g","h"); endConfig(); @@ -522,6 +535,44 @@ public class TestConfiguration extends TestCase { tearDown(); } + public void testIncludesWithFallback() throws Exception { + tearDown(); + out=new BufferedWriter(new FileWriter(CONFIG2)); + startConfig(); + appendProperty("a","b"); + appendProperty("c","d"); + endConfig(); + + out=new BufferedWriter(new FileWriter(CONFIG)); + startConfig(); + startInclude(CONFIG2); + startFallback(); + appendProperty("a", "b.fallback"); + appendProperty("c", "d.fallback", true); + endFallback(); + endInclude(); + appendProperty("e","f"); + appendProperty("g","h"); + startInclude("MissingConfig.xml"); + startFallback(); + appendProperty("i", "j.fallback"); + appendProperty("k", "l.fallback", true); + endFallback(); + endInclude(); + endConfig(); + + // verify that the includes file contains all properties + Path fileResource = new Path(CONFIG); + conf.addResource(fileResource); + assertEquals("b", conf.get("a")); + assertEquals("d", conf.get("c")); + assertEquals("f", conf.get("e")); + assertEquals("h", conf.get("g")); + assertEquals("j.fallback", conf.get("i")); + assertEquals("l.fallback", conf.get("k")); + tearDown(); + } + public void testRelativeIncludes() throws Exception { tearDown(); String relConfig = new File("./tmp/test-config.xml").getAbsolutePath(); @@ -536,7 +587,8 @@ public class TestConfiguration extends TestCase { out = new BufferedWriter(new FileWriter(relConfig)); startConfig(); // Add the relative path instead of the absolute one. - addInclude(new File(relConfig2).getName()); + startInclude(new File(relConfig2).getName()); + endInclude(); appendProperty("c", "d"); endConfig(); diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 68e8b5c296b..e55308fed19 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -869,6 +869,16 @@ core 3.1.1 + + org.codehaus.woodstox + stax2-api + 3.1.4 + + + com.fasterxml + aalto-xml + 1.0.0 + org.codehaus.jackson jackson-mapper-asl From 15e3873dc3d46016344887570e5d4aa8d20ca264 Mon Sep 17 00:00:00 2001 From: Daniel Templeton Date: Wed, 29 Mar 2017 10:56:36 -0700 Subject: [PATCH 155/188] HDFS-11571. Typo in DataStorage exception message (Contributed by Anna Budai via Daniel Templeton) --- .../org/apache/hadoop/hdfs/server/datanode/DataStorage.java | 2 +- .../org/apache/hadoop/hdfs/server/datanode/TestDataStorage.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java index 57ef1bcec99..835643bd9e7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java @@ -543,7 +543,7 @@ public class DataStorage extends Storage { void recoverTransitionRead(DataNode datanode, NamespaceInfo nsInfo, Collection dataDirs, StartupOption startOpt) throws IOException { if (addStorageLocations(datanode, nsInfo, dataDirs, startOpt).isEmpty()) { - throw new IOException("All specified directories are failed to load."); + throw new IOException("All specified directories have failed to load."); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataStorage.java index c56a01ca58e..6c494519672 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataStorage.java @@ -200,7 +200,7 @@ public class TestDataStorage { fail("An IOException should throw: all StorageLocations are NON_EXISTENT"); } catch (IOException e) { GenericTestUtils.assertExceptionContains( - "All specified directories are failed to load.", e); + "All specified directories have failed to load.", e); } assertEquals(0, storage.getNumStorageDirs()); } From 13c766b62c48ae07849f7f1762ef6883239aef54 Mon Sep 17 00:00:00 2001 From: Mingliang Liu Date: Mon, 27 Mar 2017 16:37:39 -0700 Subject: [PATCH 156/188] HADOOP-14247. FileContextMainOperationsBaseTest should clean up test root path. Contributed by Mingliang Liu --- .../fs/FileContextMainOperationsBaseTest.java | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java index 2b3ab2a0d98..ece96f855ab 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java @@ -36,6 +36,8 @@ import org.junit.Assert; import static org.junit.Assert.*; import org.junit.Before; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import static org.apache.hadoop.fs.FileContextTestHelper.*; import static org.apache.hadoop.fs.CreateFlag.*; @@ -60,6 +62,9 @@ import static org.apache.hadoop.fs.CreateFlag.*; *

    */ public abstract class FileContextMainOperationsBaseTest { + + protected static final Logger LOG = + LoggerFactory.getLogger(FileContextMainOperationsBaseTest.class); private static String TEST_DIR_AAA2 = "test/hadoop2/aaa"; private static String TEST_DIR_AAA = "test/hadoop/aaa"; @@ -111,9 +116,19 @@ public abstract class FileContextMainOperationsBaseTest { @After public void tearDown() throws Exception { if (fc != null) { - boolean del = fc.delete(new Path(fileContextTestHelper.getAbsoluteTestRootPath(fc), new Path("test")), true); - assertTrue(del); - fc.delete(localFsRootPath, true); + final Path testRoot = fileContextTestHelper.getAbsoluteTestRootPath(fc); + LOG.info("Deleting test root path {}", testRoot); + try { + fc.delete(testRoot, true); + } catch (Exception e) { + LOG.error("Error when deleting test root path " + testRoot, e); + } + + try { + fc.delete(localFsRootPath, true); + } catch (Exception e) { + LOG.error("Error when deleting localFsRootPath " + localFsRootPath, e); + } } } From 640ba1d23fe8b8105bae6d342ddc1c839302f8e5 Mon Sep 17 00:00:00 2001 From: Daniel Templeton Date: Wed, 29 Mar 2017 12:36:15 -0700 Subject: [PATCH 157/188] YARN-5685. RM configuration allows all failover methods to disabled when automatic failover is enabled --- .../org/apache/hadoop/yarn/conf/HAUtil.java | 30 +++++- .../hadoop/yarn/conf/YarnConfiguration.java | 16 +++- .../apache/hadoop/yarn/conf/TestHAUtil.java | 86 ++++++++++------- .../resourcemanager/TestRMAdminService.java | 95 +++++++++++++++++++ 4 files changed, 191 insertions(+), 36 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java index e4948e74b05..133b377c0b5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java @@ -34,6 +34,7 @@ import java.util.Collection; public class HAUtil { private static Log LOG = LogFactory.getLog(HAUtil.class); + @VisibleForTesting public static final String BAD_CONFIG_MESSAGE_PREFIX = "Invalid configuration! "; @@ -79,6 +80,7 @@ public class HAUtil { throws YarnRuntimeException { verifyAndSetRMHAIdsList(conf); verifyAndSetCurrentRMHAId(conf); + verifyLeaderElection(conf); verifyAndSetAllServiceAddresses(conf); } @@ -117,7 +119,7 @@ public class HAUtil { msg.append("Can not find valid RM_HA_ID. None of "); for (String id : conf .getTrimmedStringCollection(YarnConfiguration.RM_HA_IDS)) { - msg.append(addSuffix(YarnConfiguration.RM_ADDRESS, id) + " "); + msg.append(addSuffix(YarnConfiguration.RM_ADDRESS, id)).append(" "); } msg.append(" are matching" + " the local address OR " + YarnConfiguration.RM_HA_ID + " is not" + @@ -133,6 +135,32 @@ public class HAUtil { conf.set(YarnConfiguration.RM_HA_ID, rmId); } + /** + * This method validates that some leader election service is enabled. YARN + * allows leadership election to be disabled in the configuration, which + * breaks automatic failover. If leadership election is disabled, this + * method will throw an exception via + * {@link #throwBadConfigurationException(java.lang.String)}. + * + * @param conf the {@link Configuration} to validate + */ + private static void verifyLeaderElection(Configuration conf) { + if (isAutomaticFailoverEnabled(conf) && + !conf.getBoolean(YarnConfiguration.CURATOR_LEADER_ELECTOR, + YarnConfiguration.DEFAULT_CURATOR_LEADER_ELECTOR_ENABLED) && + !isAutomaticFailoverEmbedded(conf)) { + throwBadConfigurationException(NO_LEADER_ELECTION_MESSAGE); + } + } + + @VisibleForTesting + static final String NO_LEADER_ELECTION_MESSAGE = + "The yarn.resourcemanager.ha.automatic-failover.embedded " + + "and yarn.resourcemanager.ha.curator-leader-elector.enabled " + + "properties are both false. One of these two properties must " + + "be true when yarn.resourcemanager.ha.automatic-failover.enabled " + + "is true"; + private static void verifyAndSetConfValue(String prefix, Configuration conf) { String confKey = null; String confValue = null; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index b36685533fd..f52e487524d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -627,8 +627,22 @@ public class YarnConfiguration extends Configuration { AUTO_FAILOVER_PREFIX + "enabled"; public static final boolean DEFAULT_AUTO_FAILOVER_ENABLED = true; + /** + * This property controls whether {@link ActiveStandbyElector} leader + * election should be used when {@link #CURATOR_LEADER_ELECTOR} is + * {@code false}. + * + * @deprecated This property should never be set to {@code false}. + */ + @Deprecated public static final String AUTO_FAILOVER_EMBEDDED = AUTO_FAILOVER_PREFIX + "embedded"; + /** + * The default value for {@link #AUTO_FAILOVER_EMBEDDED}. + * + * @deprecated The {@link #AUTO_FAILOVER_EMBEDDED} property is deprecated. + */ + @Deprecated public static final boolean DEFAULT_AUTO_FAILOVER_EMBEDDED = true; public static final String AUTO_FAILOVER_ZK_BASE_PATH = @@ -667,7 +681,7 @@ public class YarnConfiguration extends Configuration { /** - * Whether to use curator-based elector for leader election. + * Whether to use the Curator-based elector for leader election. * * @deprecated Eventually, we want to default to the curator-based * implementation and remove the {@link ActiveStandbyElector} based diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java index 6ced5f26326..fc2c1d0d335 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java @@ -85,44 +85,47 @@ public class TestHAUtil { @Test public void testVerifyAndSetConfiguration() throws Exception { + Configuration myConf = new Configuration(conf); + try { - HAUtil.verifyAndSetConfiguration(conf); + HAUtil.verifyAndSetConfiguration(myConf); } catch (YarnRuntimeException e) { fail("Should not throw any exceptions."); } assertEquals("Should be saved as Trimmed collection", - StringUtils.getStringCollection(RM_NODE_IDS), HAUtil.getRMHAIds(conf)); + StringUtils.getStringCollection(RM_NODE_IDS), + HAUtil.getRMHAIds(myConf)); assertEquals("Should be saved as Trimmed string", - RM1_NODE_ID, HAUtil.getRMHAId(conf)); - for (String confKey : YarnConfiguration.getServiceAddressConfKeys(conf)) { + RM1_NODE_ID, HAUtil.getRMHAId(myConf)); + for (String confKey : YarnConfiguration.getServiceAddressConfKeys(myConf)) { assertEquals("RPC address not set for " + confKey, - RM1_ADDRESS, conf.get(confKey)); + RM1_ADDRESS, myConf.get(confKey)); } - conf.clear(); - conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID); + myConf = new Configuration(conf); + myConf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID); try { - HAUtil.verifyAndSetConfiguration(conf); + HAUtil.verifyAndSetConfiguration(myConf); } catch (YarnRuntimeException e) { assertEquals("YarnRuntimeException by verifyAndSetRMHAIds()", HAUtil.BAD_CONFIG_MESSAGE_PREFIX + HAUtil.getInvalidValueMessage(YarnConfiguration.RM_HA_IDS, - conf.get(YarnConfiguration.RM_HA_IDS) + + myConf.get(YarnConfiguration.RM_HA_IDS) + "\nHA mode requires atleast two RMs"), e.getMessage()); } - conf.clear(); + myConf = new Configuration(conf); // simulate the case YarnConfiguration.RM_HA_ID is not set - conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + myConf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID); - for (String confKey : YarnConfiguration.getServiceAddressConfKeys(conf)) { - conf.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS); - conf.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS); + for (String confKey : YarnConfiguration.getServiceAddressConfKeys(myConf)) { + myConf.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS); + myConf.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS); } try { - HAUtil.verifyAndSetConfiguration(conf); + HAUtil.verifyAndSetConfiguration(myConf); } catch (YarnRuntimeException e) { assertEquals("YarnRuntimeException by getRMId()", HAUtil.BAD_CONFIG_MESSAGE_PREFIX + @@ -130,16 +133,16 @@ public class TestHAUtil { e.getMessage()); } - conf.clear(); - conf.set(YarnConfiguration.RM_HA_ID, RM_INVALID_NODE_ID); - conf.set(YarnConfiguration.RM_HA_IDS, RM_INVALID_NODE_ID + "," + myConf = new Configuration(conf); + myConf.set(YarnConfiguration.RM_HA_ID, RM_INVALID_NODE_ID); + myConf.set(YarnConfiguration.RM_HA_IDS, RM_INVALID_NODE_ID + "," + RM1_NODE_ID); - for (String confKey : YarnConfiguration.getServiceAddressConfKeys(conf)) { + for (String confKey : YarnConfiguration.getServiceAddressConfKeys(myConf)) { // simulate xml with invalid node id - conf.set(confKey + RM_INVALID_NODE_ID, RM_INVALID_NODE_ID); + myConf.set(confKey + RM_INVALID_NODE_ID, RM_INVALID_NODE_ID); } try { - HAUtil.verifyAndSetConfiguration(conf); + HAUtil.verifyAndSetConfiguration(myConf); } catch (YarnRuntimeException e) { assertEquals("YarnRuntimeException by addSuffix()", HAUtil.BAD_CONFIG_MESSAGE_PREFIX + @@ -148,12 +151,12 @@ public class TestHAUtil { e.getMessage()); } - conf.clear(); + myConf = new Configuration(); // simulate the case HAUtil.RM_RPC_ADDRESS_CONF_KEYS are not set - conf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID); - conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID); + myConf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID); + myConf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID); try { - HAUtil.verifyAndSetConfiguration(conf); + HAUtil.verifyAndSetConfiguration(myConf); fail("Should throw YarnRuntimeException. by Configuration#set()"); } catch (YarnRuntimeException e) { String confKey = @@ -166,21 +169,36 @@ public class TestHAUtil { // simulate the case YarnConfiguration.RM_HA_IDS doesn't contain // the value of YarnConfiguration.RM_HA_ID - conf.clear(); - conf.set(YarnConfiguration.RM_HA_IDS, RM2_NODE_ID + "," + RM3_NODE_ID); - conf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID_UNTRIMMED); - for (String confKey : YarnConfiguration.getServiceAddressConfKeys(conf)) { - conf.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS_UNTRIMMED); - conf.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS); - conf.set(HAUtil.addSuffix(confKey, RM3_NODE_ID), RM3_ADDRESS); + myConf = new Configuration(conf); + myConf.set(YarnConfiguration.RM_HA_IDS, RM2_NODE_ID + "," + RM3_NODE_ID); + myConf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID_UNTRIMMED); + for (String confKey : YarnConfiguration.getServiceAddressConfKeys(myConf)) { + myConf.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS_UNTRIMMED); + myConf.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS); + myConf.set(HAUtil.addSuffix(confKey, RM3_NODE_ID), RM3_ADDRESS); } try { - HAUtil.verifyAndSetConfiguration(conf); + HAUtil.verifyAndSetConfiguration(myConf); } catch (YarnRuntimeException e) { assertEquals("YarnRuntimeException by getRMId()'s validation", HAUtil.BAD_CONFIG_MESSAGE_PREFIX + HAUtil.getRMHAIdNeedToBeIncludedMessage("[rm2, rm3]", RM1_NODE_ID), - e.getMessage()); + e.getMessage()); + } + + // simulate the case that no leader election is enabled + myConf = new Configuration(conf); + myConf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); + myConf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, true); + myConf.setBoolean(YarnConfiguration.AUTO_FAILOVER_EMBEDDED, false); + myConf.setBoolean(YarnConfiguration.CURATOR_LEADER_ELECTOR, false); + + try { + HAUtil.verifyAndSetConfiguration(myConf); + } catch (YarnRuntimeException e) { + assertEquals("YarnRuntimeException by getRMId()'s validation", + HAUtil.BAD_CONFIG_MESSAGE_PREFIX + HAUtil.NO_LEADER_ELECTION_MESSAGE, + e.getMessage()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java index 9ae28c2bf83..8d00ae071e2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java @@ -78,6 +78,8 @@ import org.junit.Test; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import static org.junit.Assert.assertTrue; public class TestRMAdminService { @@ -841,6 +843,78 @@ public class TestRMAdminService { } } + /** + * Test that a configuration with no leader election configured fails. + */ + @Test + public void testHAConfWithoutLeaderElection() { + Configuration conf = new Configuration(configuration); + + conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); + conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, true); + conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_EMBEDDED, false); + conf.setBoolean(YarnConfiguration.CURATOR_LEADER_ELECTOR, false); + conf.set(YarnConfiguration.RM_HA_IDS, "rm1,rm2"); + + int base = 100; + + for (String confKey : + YarnConfiguration.getServiceAddressConfKeys(configuration)) { + conf.set(HAUtil.addSuffix(confKey, "rm1"), "0.0.0.0:" + + (base + 20)); + conf.set(HAUtil.addSuffix(confKey, "rm2"), "0.0.0.0:" + + (base + 40)); + base = base * 2; + } + + conf.set(YarnConfiguration.RM_HA_ID, "rm1"); + + checkBadConfiguration(conf); + } + + /** + * Test that a configuration with a single RM fails. + */ + @Test + public void testHAConfWithSingleRMID() { + Configuration conf = new Configuration(configuration); + + conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); + conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, true); + conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_EMBEDDED, true); + conf.set(YarnConfiguration.RM_HA_IDS, "rm1"); + + int base = 100; + + for (String confKey : + YarnConfiguration.getServiceAddressConfKeys(configuration)) { + conf.set(HAUtil.addSuffix(confKey, "rm1"), "0.0.0.0:" + + (base + 20)); + base = base * 2; + } + + conf.set(YarnConfiguration.RM_HA_ID, "rm1"); + + checkBadConfiguration(conf); + } + + /** + * Test that a configuration with no service information fails. + */ + @Test + public void testHAConfWithoutServiceInfo() { + Configuration conf = new Configuration(configuration); + + conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); + conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, true); + conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_EMBEDDED, true); + conf.set(YarnConfiguration.RM_HA_IDS, "rm1,rm2"); + + conf.set(YarnConfiguration.RM_HA_ID, "rm1"); + + checkBadConfiguration(conf); + } + @Test public void testRMStartsWithoutConfigurationFilesProvided() { // enable FileSystemBasedConfigurationProvider without uploading @@ -1368,4 +1442,25 @@ public class TestRMAdminService { base = base * 2; } } + + /** + * This method initializes an RM with the given configuration and expects it + * to fail with a configuration error. + * + * @param conf the {@link Configuration} to use + */ + private void checkBadConfiguration(Configuration conf) { + MockRM rm1 = null; + + conf.set(YarnConfiguration.RM_HA_ID, "rm1"); + + try { + rm1 = new MockRM(conf); + rm1.init(conf); + fail("The RM allowed an invalid configuration"); + } catch (YarnRuntimeException e) { + assertTrue("The RM initialization threw an unexpected exception", + e.getMessage().startsWith(HAUtil.BAD_CONFIG_MESSAGE_PREFIX)); + } + } } From 4966a6e26e45d7dc36e0b270066ff7c87bcd00cc Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Wed, 29 Mar 2017 14:37:21 -0700 Subject: [PATCH 158/188] HADOOP-14223. Extend FileStatus#toString() to include details like Erasure Coding and Encryption. Contributed by Manoj Govindassamy. --- .../java/org/apache/hadoop/fs/FileStatus.java | 12 ++++++++++ .../site/markdown/filesystem/filesystem.md | 19 ++++++--------- .../org/apache/hadoop/fs/TestFileStatus.java | 10 +++++--- .../fs/viewfs/TestViewfsFileStatus.java | 3 +++ .../apache/hadoop/hdfs/TestFileStatus.java | 3 +++ .../hdfs/TestFileStatusWithECPolicy.java | 7 +++++- .../hdfs/server/namenode/FSAclBaseTest.java | 23 +++++++++++++++++++ .../apache/hadoop/hdfs/web/TestWebHDFS.java | 12 ++++++++++ .../hadoop/fs/s3a/TestS3AGetFileStatus.java | 6 +++++ .../hadoop/fs/adl/TestGetFileStatus.java | 7 ++++++ 10 files changed, 86 insertions(+), 16 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java index 26d39627f50..f5111ef6f0a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java @@ -207,6 +207,15 @@ public class FileStatus implements Writable, Comparable, return permission; } + /** + * Tell whether the underlying file or directory has ACLs set. + * + * @return true if the underlying file or directory has ACLs set. + */ + public boolean hasAcl() { + return permission.getAclBit(); + } + /** * Tell whether the underlying file or directory is encrypted or not. * @@ -399,6 +408,9 @@ public class FileStatus implements Writable, Comparable, if(isSymlink()) { sb.append("; symlink=" + symlink); } + sb.append("; hasAcl=" + hasAcl()); + sb.append("; isEncrypted=" + isEncrypted()); + sb.append("; isErasureCoded=" + isErasureCoded()); sb.append("}"); return sb.toString(); } diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md index 97bc7d19dfd..b464941537b 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md @@ -86,20 +86,15 @@ Get the status of a path stat.length = 0 stat.isdir = False stat.symlink = FS.Symlinks[p] - if inEncryptionZone(FS, p) : - stat.isEncrypted = True - else - stat.isEncrypted = False - if isErasureCoded(FS, p) : - stat.isErasureCoded = True - else - stat.isErasureCoded = False + stat.hasAcl = hasACL(FS, p) + stat.isEncrypted = inEncryptionZone(FS, p) + stat.isErasureCoded = isErasureCoded(FS, p) The returned `FileStatus` status of the path additionally carries details on -Encryption and Erasure Coding information. `getFileStatus(Path p).isEncrypted()` -can be queried to find if the path is Encrypted. -Likewise, `getFileStatus(Path p).isErasureCoded()` will tell if the path is -Erasure Coded or not. +ACL, encryption and erasure coding information. `getFileStatus(Path p).hasAcl()` +can be queried to find if the path has an ACL. `getFileStatus(Path p).isEncrypted()` +can be queried to find if the path is encrypted. `getFileStatus(Path p).isErasureCoded()` +will tell if the path is erasure coded or not. ### `Path getHomeDirectory()` diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java index 35f2bad5657..d29b1a40711 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java @@ -295,11 +295,15 @@ public class TestFileStatus { expected.append("permission=").append(fileStatus.getPermission()).append("; "); if(fileStatus.isSymlink()) { expected.append("isSymlink=").append(true).append("; "); - expected.append("symlink=").append(fileStatus.getSymlink()).append("}"); + expected.append("symlink=").append(fileStatus.getSymlink()).append("; "); } else { - expected.append("isSymlink=").append(false).append("}"); + expected.append("isSymlink=").append(false).append("; "); } - + expected.append("hasAcl=").append(fileStatus.hasAcl()).append("; "); + expected.append("isEncrypted=").append( + fileStatus.isEncrypted()).append("; "); + expected.append("isErasureCoded=").append( + fileStatus.isErasureCoded()).append("}"); assertEquals(expected.toString(), fileStatus.toString()); } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewfsFileStatus.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewfsFileStatus.java index 449e2e33621..0c31c8ed6a9 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewfsFileStatus.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewfsFileStatus.java @@ -75,6 +75,9 @@ public class TestViewfsFileStatus { FileStatus stat = vfs.getFileStatus(path); assertEquals(content.length, stat.getLen()); ContractTestUtils.assertNotErasureCoded(vfs, path); + assertTrue(path + " should have erasure coding unset in " + + "FileStatus#toString(): " + stat, + stat.toString().contains("isErasureCoded=false")); // check serialization/deserialization DataOutputBuffer dob = new DataOutputBuffer(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java index 7631e7d70cc..c74bb632a74 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java @@ -139,6 +139,9 @@ public class TestFileStatus { assertEquals(file1.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString(), status.getPath().toString()); + assertTrue(file1 + " should have erasure coding unset in " + + "FileStatus#toString(): " + status, + status.toString().contains("isErasureCoded=false")); } /** Test the FileStatus obtained calling listStatus on a file */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java index 0e270ff75c7..e04f9573256 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java @@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue; import java.io.IOException; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.fs.permission.FsPermission; @@ -85,7 +86,7 @@ public class TestFileStatusWithECPolicy { assertNotNull(ecPolicy2); assertTrue(ecPolicy1.equals(ecPolicy2)); - // test file doesn't have an EC policy + // test file with EC policy fs.create(file).close(); final ErasureCodingPolicy ecPolicy3 = fs.getClient().getFileInfo(file.toUri().getPath()) @@ -93,5 +94,9 @@ public class TestFileStatusWithECPolicy { assertNotNull(ecPolicy3); assertTrue(ecPolicy1.equals(ecPolicy3)); ContractTestUtils.assertErasureCoded(fs, file); + FileStatus status = fs.getFileStatus(file); + assertTrue(file + " should have erasure coding set in " + + "FileStatus#toString(): " + status, + status.toString().contains("isErasureCoded=true")); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java index 81270e6f2bd..60b0ab168d2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java @@ -50,6 +50,7 @@ import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.junit.After; import org.junit.AfterClass; +import org.junit.Assert; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -120,10 +121,16 @@ public abstract class FSAclBaseTest { aclEntry(ACCESS, OTHER, NONE), aclEntry(DEFAULT, USER, "foo", ALL)); fs.setAcl(path, aclSpec); + Assert.assertTrue(path + " should have ACLs in FileStatus!", + fs.getFileStatus(path).hasAcl()); + aclSpec = Lists.newArrayList( aclEntry(ACCESS, USER, "foo", READ_EXECUTE), aclEntry(DEFAULT, USER, "foo", READ_EXECUTE)); fs.modifyAclEntries(path, aclSpec); + Assert.assertTrue(path + " should have ACLs in FileStatus!", + fs.getFileStatus(path).hasAcl()); + AclStatus s = fs.getAclStatus(path); AclEntry[] returned = s.getEntries().toArray(new AclEntry[0]); assertArrayEquals(new AclEntry[] { @@ -561,8 +568,18 @@ public abstract class FSAclBaseTest { aclEntry(ACCESS, GROUP, READ_EXECUTE), aclEntry(ACCESS, OTHER, NONE), aclEntry(DEFAULT, USER, "foo", ALL)); + fs.setAcl(path, aclSpec); + Assert.assertTrue(path + " should have ACLs in FileStatus!", + fs.getFileStatus(path).hasAcl()); + Assert.assertTrue(path + " should have ACLs in FileStatus#toString()!", + fs.getFileStatus(path).toString().contains("hasAcl=true")); fs.removeAcl(path); + Assert.assertFalse(path + " should not have ACLs in FileStatus!", + fs.getFileStatus(path).hasAcl()); + Assert.assertTrue(path + " should not have ACLs in FileStatus#toString()!", + fs.getFileStatus(path).toString().contains("hasAcl=false")); + AclStatus s = fs.getAclStatus(path); AclEntry[] returned = s.getEntries().toArray(new AclEntry[0]); assertArrayEquals(new AclEntry[] { }, returned); @@ -968,8 +985,14 @@ public abstract class FSAclBaseTest { List aclSpec = Lists.newArrayList( aclEntry(DEFAULT, USER, "foo", ALL)); fs.setAcl(path, aclSpec); + Assert.assertTrue(path + " should have ACLs in FileStatus!", + fs.getFileStatus(path).hasAcl()); + Path dirPath = new Path(path, "dir1"); fs.mkdirs(dirPath); + Assert.assertTrue(dirPath + " should have ACLs in FileStatus!", + fs.getFileStatus(dirPath).hasAcl()); + AclStatus s = fs.getAclStatus(dirPath); AclEntry[] returned = s.getEntries().toArray(new AclEntry[0]); assertArrayEquals(new AclEntry[] { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java index 3c366c39dad..46a8a56f1c1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java @@ -547,24 +547,36 @@ public class TestWebHDFS { Assert.assertEquals(expectedECDirStatus.isErasureCoded(), actualECDirStatus.isErasureCoded()); ContractTestUtils.assertErasureCoded(dfs, ecDir); + assertTrue(ecDir+ " should have erasure coding set in " + + "FileStatus#toString(): " + actualECDirStatus, + actualECDirStatus.toString().contains("isErasureCoded=true")); FileStatus expectedECFileStatus = dfs.getFileStatus(ecFile); FileStatus actualECFileStatus = webHdfs.getFileStatus(ecFile); Assert.assertEquals(expectedECFileStatus.isErasureCoded(), actualECFileStatus.isErasureCoded()); ContractTestUtils.assertErasureCoded(dfs, ecFile); + assertTrue(ecFile+ " should have erasure coding set in " + + "FileStatus#toString(): " + actualECFileStatus, + actualECFileStatus.toString().contains("isErasureCoded=true")); FileStatus expectedNormalDirStatus = dfs.getFileStatus(normalDir); FileStatus actualNormalDirStatus = webHdfs.getFileStatus(normalDir); Assert.assertEquals(expectedNormalDirStatus.isErasureCoded(), actualNormalDirStatus.isErasureCoded()); ContractTestUtils.assertNotErasureCoded(dfs, normalDir); + assertTrue(normalDir + " should have erasure coding unset in " + + "FileStatus#toString(): " + actualNormalDirStatus, + actualNormalDirStatus.toString().contains("isErasureCoded=false")); FileStatus expectedNormalFileStatus = dfs.getFileStatus(normalFile); FileStatus actualNormalFileStatus = webHdfs.getFileStatus(normalDir); Assert.assertEquals(expectedNormalFileStatus.isErasureCoded(), actualNormalFileStatus.isErasureCoded()); ContractTestUtils.assertNotErasureCoded(dfs, normalFile); + assertTrue(normalFile + " should have erasure coding unset in " + + "FileStatus#toString(): " + actualNormalFileStatus, + actualNormalFileStatus.toString().contains("isErasureCoded=false")); } finally { if (cluster != null) { diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java index deb3963b2d2..58e4d3074bc 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java @@ -62,6 +62,9 @@ public class TestS3AGetFileStatus extends AbstractS3AMockTest { assertEquals(meta.getContentLength(), stat.getLen()); assertEquals(meta.getLastModified().getTime(), stat.getModificationTime()); ContractTestUtils.assertNotErasureCoded(fs, path); + assertTrue(path + " should have erasure coding unset in " + + "FileStatus#toString(): " + stat, + stat.toString().contains("isErasureCoded=false")); } @Test @@ -101,6 +104,9 @@ public class TestS3AGetFileStatus extends AbstractS3AMockTest { assertEquals(fs.makeQualified(path), stat.getPath()); assertTrue(stat.isDirectory()); ContractTestUtils.assertNotErasureCoded(fs, path); + assertTrue(path + " should have erasure coding unset in " + + "FileStatus#toString(): " + stat, + stat.toString().contains("isErasureCoded=false")); } @Test diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java index df39a40cebd..0ea4b868c1d 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java @@ -65,6 +65,9 @@ public class TestGetFileStatus extends AdlMockWebServer { Assert.assertEquals(new FsPermission("777"), fileStatus.getPermission()); Assert.assertEquals("NotSupportYet", fileStatus.getOwner()); Assert.assertEquals("NotSupportYet", fileStatus.getGroup()); + Assert.assertTrue(path + " should have Acl!", fileStatus.hasAcl()); + Assert.assertFalse(path + " should not be encrypted!", + fileStatus.isEncrypted()); Assert.assertFalse(path + " should not be erasure coded!", fileStatus.isErasureCoded()); } @@ -82,6 +85,8 @@ public class TestGetFileStatus extends AdlMockWebServer { LOG.debug("Time : " + (endTime - startTime)); Assert.assertTrue(fileStatus.isFile()); Assert.assertEquals(true, fileStatus.getPermission().getAclBit()); + Assert.assertEquals(fileStatus.hasAcl(), + fileStatus.getPermission().getAclBit()); // With ACLBIT set to false getMockServer().enqueue(new MockResponse().setResponseCode(200) @@ -93,5 +98,7 @@ public class TestGetFileStatus extends AdlMockWebServer { LOG.debug("Time : " + (endTime - startTime)); Assert.assertTrue(fileStatus.isFile()); Assert.assertEquals(false, fileStatus.getPermission().getAclBit()); + Assert.assertEquals(fileStatus.hasAcl(), + fileStatus.getPermission().getAclBit()); } } From 6a5516c2381f107d96b8326939514de3c6e53d3d Mon Sep 17 00:00:00 2001 From: Robert Kanter Date: Wed, 29 Mar 2017 16:18:13 -0700 Subject: [PATCH 159/188] YARN-5654. Not be able to run SLS with FairScheduler (yufeigu via rkanter) --- .../org/apache/hadoop/yarn/sls/SLSRunner.java | 22 +- .../yarn/sls/appmaster/AMSimulator.java | 24 +- .../scheduler/ResourceSchedulerWrapper.java | 969 ------------------ .../sls/scheduler/SLSCapacityScheduler.java | 685 ++----------- .../yarn/sls/scheduler/SLSFairScheduler.java | 339 ++++++ .../yarn/sls/scheduler/SchedulerMetrics.java | 533 +++++++++- .../yarn/sls/scheduler/SchedulerWrapper.java | 18 +- .../hadoop/yarn/sls/scheduler/Tracker.java | 46 + .../hadoop/yarn/sls/utils/SLSUtils.java | 1 + .../apache/hadoop/yarn/sls/web/SLSWebApp.java | 12 +- .../yarn/sls/appmaster/TestAMSimulator.java | 53 +- .../yarn/sls/nodemanager/TestNMSimulator.java | 32 +- 12 files changed, 1103 insertions(+), 1631 deletions(-) delete mode 100644 hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java create mode 100644 hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java create mode 100644 hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/Tracker.java diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java index 61b7f36ee9c..ba43816de19 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java @@ -59,16 +59,14 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.sls.appmaster.AMSimulator; import org.apache.hadoop.yarn.sls.conf.SLSConfiguration; import org.apache.hadoop.yarn.sls.nodemanager.NMSimulator; import org.apache.hadoop.yarn.sls.resourcemanager.MockAMLauncher; -import org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator; -import org.apache.hadoop.yarn.sls.scheduler.ResourceSchedulerWrapper; -import org.apache.hadoop.yarn.sls.scheduler.SLSCapacityScheduler; -import org.apache.hadoop.yarn.sls.scheduler.SchedulerWrapper; -import org.apache.hadoop.yarn.sls.scheduler.TaskRunner; +import org.apache.hadoop.yarn.sls.scheduler.*; import org.apache.hadoop.yarn.sls.utils.SLSUtils; import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.log4j.Logger; @@ -152,9 +150,9 @@ public class SLSRunner { // start application masters startAM(); // set queue & tracked apps information - ((SchedulerWrapper) rm.getResourceScheduler()) + ((SchedulerWrapper) rm.getResourceScheduler()).getTracker() .setQueueSet(this.queueAppNumMap.keySet()); - ((SchedulerWrapper) rm.getResourceScheduler()) + ((SchedulerWrapper) rm.getResourceScheduler()).getTracker() .setTrackedAppSet(this.trackedApps); // print out simulation info printSimulationInfo(); @@ -164,7 +162,7 @@ public class SLSRunner { runner.start(); } - private void startRM() throws IOException, ClassNotFoundException { + private void startRM() throws Exception { Configuration rmConf = new YarnConfiguration(); String schedulerClass = rmConf.get(YarnConfiguration.RM_SCHEDULER); @@ -175,10 +173,12 @@ public class SLSRunner { if(Class.forName(schedulerClass) == CapacityScheduler.class) { rmConf.set(YarnConfiguration.RM_SCHEDULER, SLSCapacityScheduler.class.getName()); - } else { + } else if (Class.forName(schedulerClass) == FairScheduler.class) { rmConf.set(YarnConfiguration.RM_SCHEDULER, - ResourceSchedulerWrapper.class.getName()); - rmConf.set(SLSConfiguration.RM_SCHEDULER, schedulerClass); + SLSFairScheduler.class.getName()); + } else if (Class.forName(schedulerClass) == FifoScheduler.class){ + // TODO add support for FifoScheduler + throw new Exception("Fifo Scheduler is not supported yet."); } rmConf.set(SLSConfiguration.METRICS_OUTPUT_DIR, metricsOutputDir); diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java index 0573bae0ecf..a62f2b60240 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java @@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.sls.scheduler.SchedulerMetrics; import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.log4j.Logger; @@ -219,10 +220,12 @@ public abstract class AMSimulator extends TaskRunner.Task { simulateFinishTimeMS = System.currentTimeMillis() - SLSRunner.getRunner().getStartTimeMS(); // record job running information - ((SchedulerWrapper)rm.getResourceScheduler()) - .addAMRuntime(appId, - traceStartTimeMS, traceFinishTimeMS, - simulateStartTimeMS, simulateFinishTimeMS); + SchedulerMetrics schedulerMetrics = + ((SchedulerWrapper)rm.getResourceScheduler()).getSchedulerMetrics(); + if (schedulerMetrics != null) { + schedulerMetrics.addAMRuntime(appId, traceStartTimeMS, traceFinishTimeMS, + simulateStartTimeMS, simulateFinishTimeMS); + } } protected ResourceRequest createResourceRequest( @@ -330,13 +333,20 @@ public abstract class AMSimulator extends TaskRunner.Task { private void trackApp() { if (isTracked) { - ((SchedulerWrapper) rm.getResourceScheduler()) - .addTrackedApp(appId, oldAppId); + SchedulerMetrics schedulerMetrics = + ((SchedulerWrapper)rm.getResourceScheduler()).getSchedulerMetrics(); + if (schedulerMetrics != null) { + schedulerMetrics.addTrackedApp(appId, oldAppId); + } } } public void untrackApp() { if (isTracked) { - ((SchedulerWrapper) rm.getResourceScheduler()).removeTrackedApp(oldAppId); + SchedulerMetrics schedulerMetrics = + ((SchedulerWrapper)rm.getResourceScheduler()).getSchedulerMetrics(); + if (schedulerMetrics != null) { + schedulerMetrics.removeTrackedApp(oldAppId); + } } } diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java deleted file mode 100644 index a4b8e642ba5..00000000000 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java +++ /dev/null @@ -1,969 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.sls.scheduler; - -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; -import java.util.SortedMap; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; - -import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate; -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.util.ReflectionUtils; -import org.apache.hadoop.util.ShutdownHookManager; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.ContainerExitStatus; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.NodeId; -import org.apache.hadoop.yarn.api.records.Priority; -import org.apache.hadoop.yarn.api.records.QueueACL; -import org.apache.hadoop.yarn.api.records.QueueInfo; -import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.api.records.ResourceRequest; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.server.resourcemanager.RMContext; -import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; -import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; -import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; -import org.apache.hadoop.yarn.sls.SLSRunner; -import org.apache.hadoop.yarn.sls.conf.SLSConfiguration; -import org.apache.hadoop.yarn.sls.web.SLSWebApp; -import org.apache.hadoop.yarn.util.resource.ResourceCalculator; -import org.apache.hadoop.yarn.util.resource.Resources; -import org.apache.log4j.Logger; - -import com.codahale.metrics.Counter; -import com.codahale.metrics.CsvReporter; -import com.codahale.metrics.Gauge; -import com.codahale.metrics.Histogram; -import com.codahale.metrics.MetricRegistry; -import com.codahale.metrics.SlidingWindowReservoir; -import com.codahale.metrics.Timer; - -@Private -@Unstable -final public class ResourceSchedulerWrapper - extends AbstractYarnScheduler - implements SchedulerWrapper, ResourceScheduler, Configurable { - private static final String EOL = System.getProperty("line.separator"); - private static final int SAMPLING_SIZE = 60; - private ScheduledExecutorService pool; - // counters for scheduler allocate/handle operations - private Counter schedulerAllocateCounter; - private Counter schedulerHandleCounter; - private Map schedulerHandleCounterMap; - // Timers for scheduler allocate/handle operations - private Timer schedulerAllocateTimer; - private Timer schedulerHandleTimer; - private Map schedulerHandleTimerMap; - private List schedulerHistogramList; - private Map histogramTimerMap; - private Lock samplerLock; - private Lock queueLock; - - private Configuration conf; - private ResourceScheduler scheduler; - private Map appQueueMap = - new ConcurrentHashMap(); - private BufferedWriter jobRuntimeLogBW; - - // Priority of the ResourceSchedulerWrapper shutdown hook. - public static final int SHUTDOWN_HOOK_PRIORITY = 30; - - // web app - private SLSWebApp web; - - private Map preemptionContainerMap = - new ConcurrentHashMap(); - - // metrics - private MetricRegistry metrics; - private SchedulerMetrics schedulerMetrics; - private boolean metricsON; - private String metricsOutputDir; - private BufferedWriter metricsLogBW; - private boolean running = false; - private static Map defaultSchedulerMetricsMap = - new HashMap(); - static { - defaultSchedulerMetricsMap.put(FairScheduler.class, - FairSchedulerMetrics.class); - defaultSchedulerMetricsMap.put(FifoScheduler.class, - FifoSchedulerMetrics.class); - defaultSchedulerMetricsMap.put(CapacityScheduler.class, - CapacitySchedulerMetrics.class); - } - // must set by outside - private Set queueSet; - private Set trackedAppSet; - - public final Logger LOG = Logger.getLogger(ResourceSchedulerWrapper.class); - - public ResourceSchedulerWrapper() { - super(ResourceSchedulerWrapper.class.getName()); - samplerLock = new ReentrantLock(); - queueLock = new ReentrantLock(); - } - - @Override - public void setConf(Configuration conf) { - this.conf = conf; - // set scheduler - Class klass = conf.getClass( - SLSConfiguration.RM_SCHEDULER, null, ResourceScheduler.class); - - scheduler = ReflectionUtils.newInstance(klass, conf); - // start metrics - metricsON = conf.getBoolean(SLSConfiguration.METRICS_SWITCH, true); - if (metricsON) { - try { - initMetrics(); - } catch (Exception e) { - e.printStackTrace(); - } - } - - ShutdownHookManager.get().addShutdownHook(new Runnable() { - @Override - public void run() { - try { - if (metricsLogBW != null) { - metricsLogBW.write("]"); - metricsLogBW.close(); - } - if (web != null) { - web.stop(); - } - tearDown(); - } catch (Exception e) { - e.printStackTrace(); - } - } - }, SHUTDOWN_HOOK_PRIORITY); - } - - @Override - public Allocation allocate(ApplicationAttemptId attemptId, - List resourceRequests, List containerIds, - List strings, List strings2, - ContainerUpdates updateRequests) { - if (metricsON) { - final Timer.Context context = schedulerAllocateTimer.time(); - Allocation allocation = null; - try { - allocation = scheduler.allocate(attemptId, resourceRequests, - containerIds, strings, strings2, updateRequests); - return allocation; - } finally { - context.stop(); - schedulerAllocateCounter.inc(); - try { - updateQueueWithAllocateRequest(allocation, attemptId, - resourceRequests, containerIds); - } catch (IOException e) { - e.printStackTrace(); - } - } - } else { - return scheduler.allocate(attemptId, - resourceRequests, containerIds, strings, strings2, updateRequests); - } - } - - @Override - public void handle(SchedulerEvent schedulerEvent) { - // metrics off - if (! metricsON) { - scheduler.handle(schedulerEvent); - return; - } - if(!running) running = true; - - // metrics on - Timer.Context handlerTimer = null; - Timer.Context operationTimer = null; - - NodeUpdateSchedulerEventWrapper eventWrapper; - try { - //if (schedulerEvent instanceof NodeUpdateSchedulerEvent) { - if (schedulerEvent.getType() == SchedulerEventType.NODE_UPDATE - && schedulerEvent instanceof NodeUpdateSchedulerEvent) { - eventWrapper = new NodeUpdateSchedulerEventWrapper( - (NodeUpdateSchedulerEvent)schedulerEvent); - schedulerEvent = eventWrapper; - updateQueueWithNodeUpdate(eventWrapper); - } else if (schedulerEvent.getType() == SchedulerEventType.APP_ATTEMPT_REMOVED - && schedulerEvent instanceof AppAttemptRemovedSchedulerEvent) { - // check if having AM Container, update resource usage information - AppAttemptRemovedSchedulerEvent appRemoveEvent = - (AppAttemptRemovedSchedulerEvent) schedulerEvent; - ApplicationAttemptId appAttemptId = - appRemoveEvent.getApplicationAttemptID(); - String queue = appQueueMap.get(appAttemptId.getApplicationId()); - SchedulerAppReport app = scheduler.getSchedulerAppInfo(appAttemptId); - if (! app.getLiveContainers().isEmpty()) { // have 0 or 1 - // should have one container which is AM container - RMContainer rmc = app.getLiveContainers().iterator().next(); - updateQueueMetrics(queue, - rmc.getContainer().getResource().getMemorySize(), - rmc.getContainer().getResource().getVirtualCores()); - } - } - - handlerTimer = schedulerHandleTimer.time(); - operationTimer = schedulerHandleTimerMap - .get(schedulerEvent.getType()).time(); - - scheduler.handle(schedulerEvent); - } finally { - if (handlerTimer != null) handlerTimer.stop(); - if (operationTimer != null) operationTimer.stop(); - schedulerHandleCounter.inc(); - schedulerHandleCounterMap.get(schedulerEvent.getType()).inc(); - - if (schedulerEvent.getType() == SchedulerEventType.APP_REMOVED - && schedulerEvent instanceof AppRemovedSchedulerEvent) { - SLSRunner.decreaseRemainingApps(); - AppRemovedSchedulerEvent appRemoveEvent = - (AppRemovedSchedulerEvent) schedulerEvent; - appQueueMap.remove(appRemoveEvent.getApplicationID()); - } else if (schedulerEvent.getType() == SchedulerEventType.APP_ADDED - && schedulerEvent instanceof AppAddedSchedulerEvent) { - AppAddedSchedulerEvent appAddEvent = - (AppAddedSchedulerEvent) schedulerEvent; - String queueName = appAddEvent.getQueue(); - appQueueMap.put(appAddEvent.getApplicationId(), queueName); - } - } - } - - private void updateQueueWithNodeUpdate( - NodeUpdateSchedulerEventWrapper eventWrapper) { - RMNodeWrapper node = (RMNodeWrapper) eventWrapper.getRMNode(); - List containerList = node.getContainerUpdates(); - for (UpdatedContainerInfo info : containerList) { - for (ContainerStatus status : info.getCompletedContainers()) { - ContainerId containerId = status.getContainerId(); - SchedulerAppReport app = scheduler.getSchedulerAppInfo( - containerId.getApplicationAttemptId()); - - if (app == null) { - // this happens for the AM container - // The app have already removed when the NM sends the release - // information. - continue; - } - - String queue = - appQueueMap.get(containerId.getApplicationAttemptId() - .getApplicationId()); - int releasedMemory = 0, releasedVCores = 0; - if (status.getExitStatus() == ContainerExitStatus.SUCCESS) { - for (RMContainer rmc : app.getLiveContainers()) { - if (rmc.getContainerId() == containerId) { - releasedMemory += rmc.getContainer().getResource().getMemorySize(); - releasedVCores += rmc.getContainer() - .getResource().getVirtualCores(); - break; - } - } - } else if (status.getExitStatus() == ContainerExitStatus.ABORTED) { - if (preemptionContainerMap.containsKey(containerId)) { - Resource preResource = preemptionContainerMap.get(containerId); - releasedMemory += preResource.getMemorySize(); - releasedVCores += preResource.getVirtualCores(); - preemptionContainerMap.remove(containerId); - } - } - // update queue counters - updateQueueMetrics(queue, releasedMemory, releasedVCores); - } - } - } - - private void updateQueueWithAllocateRequest(Allocation allocation, - ApplicationAttemptId attemptId, - List resourceRequests, - List containerIds) throws IOException { - // update queue information - Resource pendingResource = Resources.createResource(0, 0); - Resource allocatedResource = Resources.createResource(0, 0); - String queueName = appQueueMap.get(attemptId.getApplicationId()); - // container requested - for (ResourceRequest request : resourceRequests) { - if (request.getResourceName().equals(ResourceRequest.ANY)) { - Resources.addTo(pendingResource, - Resources.multiply(request.getCapability(), - request.getNumContainers())); - } - } - // container allocated - for (Container container : allocation.getContainers()) { - Resources.addTo(allocatedResource, container.getResource()); - Resources.subtractFrom(pendingResource, container.getResource()); - } - // container released from AM - SchedulerAppReport report = scheduler.getSchedulerAppInfo(attemptId); - for (ContainerId containerId : containerIds) { - Container container = null; - for (RMContainer c : report.getLiveContainers()) { - if (c.getContainerId().equals(containerId)) { - container = c.getContainer(); - break; - } - } - if (container != null) { - // released allocated containers - Resources.subtractFrom(allocatedResource, container.getResource()); - } else { - for (RMContainer c : report.getReservedContainers()) { - if (c.getContainerId().equals(containerId)) { - container = c.getContainer(); - break; - } - } - if (container != null) { - // released reserved containers - Resources.subtractFrom(pendingResource, container.getResource()); - } - } - } - // containers released/preemption from scheduler - Set preemptionContainers = new HashSet(); - if (allocation.getContainerPreemptions() != null) { - preemptionContainers.addAll(allocation.getContainerPreemptions()); - } - if (allocation.getStrictContainerPreemptions() != null) { - preemptionContainers.addAll(allocation.getStrictContainerPreemptions()); - } - if (! preemptionContainers.isEmpty()) { - for (ContainerId containerId : preemptionContainers) { - if (! preemptionContainerMap.containsKey(containerId)) { - Container container = null; - for (RMContainer c : report.getLiveContainers()) { - if (c.getContainerId().equals(containerId)) { - container = c.getContainer(); - break; - } - } - if (container != null) { - preemptionContainerMap.put(containerId, container.getResource()); - } - } - - } - } - - // update metrics - SortedMap counterMap = metrics.getCounters(); - String names[] = new String[]{ - "counter.queue." + queueName + ".pending.memory", - "counter.queue." + queueName + ".pending.cores", - "counter.queue." + queueName + ".allocated.memory", - "counter.queue." + queueName + ".allocated.cores"}; - long values[] = new long[]{pendingResource.getMemorySize(), - pendingResource.getVirtualCores(), - allocatedResource.getMemorySize(), allocatedResource.getVirtualCores()}; - for (int i = names.length - 1; i >= 0; i --) { - if (! counterMap.containsKey(names[i])) { - metrics.counter(names[i]); - counterMap = metrics.getCounters(); - } - counterMap.get(names[i]).inc(values[i]); - } - - queueLock.lock(); - try { - if (! schedulerMetrics.isTracked(queueName)) { - schedulerMetrics.trackQueue(queueName); - } - } finally { - queueLock.unlock(); - } - } - - private void tearDown() throws IOException { - // close job runtime writer - if (jobRuntimeLogBW != null) { - jobRuntimeLogBW.close(); - } - // shut pool - if (pool != null) pool.shutdown(); - } - - @SuppressWarnings("unchecked") - private void initMetrics() throws Exception { - metrics = new MetricRegistry(); - // configuration - metricsOutputDir = conf.get(SLSConfiguration.METRICS_OUTPUT_DIR); - int metricsWebAddressPort = conf.getInt( - SLSConfiguration.METRICS_WEB_ADDRESS_PORT, - SLSConfiguration.METRICS_WEB_ADDRESS_PORT_DEFAULT); - // create SchedulerMetrics for current scheduler - String schedulerMetricsType = conf.get(scheduler.getClass().getName()); - Class schedulerMetricsClass = schedulerMetricsType == null? - defaultSchedulerMetricsMap.get(scheduler.getClass()) : - Class.forName(schedulerMetricsType); - schedulerMetrics = (SchedulerMetrics)ReflectionUtils - .newInstance(schedulerMetricsClass, new Configuration()); - schedulerMetrics.init(scheduler, metrics); - - // register various metrics - registerJvmMetrics(); - registerClusterResourceMetrics(); - registerContainerAppNumMetrics(); - registerSchedulerMetrics(); - - // .csv output - initMetricsCSVOutput(); - - // start web app to provide real-time tracking - web = new SLSWebApp(this, metricsWebAddressPort); - web.start(); - - // a thread to update histogram timer - pool = new ScheduledThreadPoolExecutor(2); - pool.scheduleAtFixedRate(new HistogramsRunnable(), 0, 1000, - TimeUnit.MILLISECONDS); - - // a thread to output metrics for real-tiem tracking - pool.scheduleAtFixedRate(new MetricsLogRunnable(), 0, 1000, - TimeUnit.MILLISECONDS); - - // application running information - jobRuntimeLogBW = - new BufferedWriter(new OutputStreamWriter(new FileOutputStream( - metricsOutputDir + "/jobruntime.csv"), "UTF-8")); - jobRuntimeLogBW.write("JobID,real_start_time,real_end_time," + - "simulate_start_time,simulate_end_time" + EOL); - jobRuntimeLogBW.flush(); - } - - private void registerJvmMetrics() { - // add JVM gauges - metrics.register("variable.jvm.free.memory", - new Gauge() { - @Override - public Long getValue() { - return Runtime.getRuntime().freeMemory(); - } - } - ); - metrics.register("variable.jvm.max.memory", - new Gauge() { - @Override - public Long getValue() { - return Runtime.getRuntime().maxMemory(); - } - } - ); - metrics.register("variable.jvm.total.memory", - new Gauge() { - @Override - public Long getValue() { - return Runtime.getRuntime().totalMemory(); - } - } - ); - } - - private void registerClusterResourceMetrics() { - metrics.register("variable.cluster.allocated.memory", - new Gauge() { - @Override - public Long getValue() { - if(scheduler == null || scheduler.getRootQueueMetrics() == null) { - return 0L; - } else { - return scheduler.getRootQueueMetrics().getAllocatedMB(); - } - } - } - ); - metrics.register("variable.cluster.allocated.vcores", - new Gauge() { - @Override - public Integer getValue() { - if(scheduler == null || scheduler.getRootQueueMetrics() == null) { - return 0; - } else { - return scheduler.getRootQueueMetrics().getAllocatedVirtualCores(); - } - } - } - ); - metrics.register("variable.cluster.available.memory", - new Gauge() { - @Override - public Long getValue() { - if(scheduler == null || scheduler.getRootQueueMetrics() == null) { - return 0L; - } else { - return scheduler.getRootQueueMetrics().getAvailableMB(); - } - } - } - ); - metrics.register("variable.cluster.available.vcores", - new Gauge() { - @Override - public Integer getValue() { - if(scheduler == null || scheduler.getRootQueueMetrics() == null) { - return 0; - } else { - return scheduler.getRootQueueMetrics().getAvailableVirtualCores(); - } - } - } - ); - } - - private void registerContainerAppNumMetrics() { - metrics.register("variable.running.application", - new Gauge() { - @Override - public Integer getValue() { - if (scheduler == null || scheduler.getRootQueueMetrics() == null) { - return 0; - } else { - return scheduler.getRootQueueMetrics().getAppsRunning(); - } - } - } - ); - metrics.register("variable.running.container", - new Gauge() { - @Override - public Integer getValue() { - if(scheduler == null || scheduler.getRootQueueMetrics() == null) { - return 0; - } else { - return scheduler.getRootQueueMetrics().getAllocatedContainers(); - } - } - } - ); - } - - private void registerSchedulerMetrics() { - samplerLock.lock(); - try { - // counters for scheduler operations - schedulerAllocateCounter = metrics.counter( - "counter.scheduler.operation.allocate"); - schedulerHandleCounter = metrics.counter( - "counter.scheduler.operation.handle"); - schedulerHandleCounterMap = new HashMap(); - for (SchedulerEventType e : SchedulerEventType.values()) { - Counter counter = metrics.counter( - "counter.scheduler.operation.handle." + e); - schedulerHandleCounterMap.put(e, counter); - } - // timers for scheduler operations - int timeWindowSize = conf.getInt( - SLSConfiguration.METRICS_TIMER_WINDOW_SIZE, - SLSConfiguration.METRICS_TIMER_WINDOW_SIZE_DEFAULT); - schedulerAllocateTimer = new Timer( - new SlidingWindowReservoir(timeWindowSize)); - schedulerHandleTimer = new Timer( - new SlidingWindowReservoir(timeWindowSize)); - schedulerHandleTimerMap = new HashMap(); - for (SchedulerEventType e : SchedulerEventType.values()) { - Timer timer = new Timer(new SlidingWindowReservoir(timeWindowSize)); - schedulerHandleTimerMap.put(e, timer); - } - // histogram for scheduler operations (Samplers) - schedulerHistogramList = new ArrayList(); - histogramTimerMap = new HashMap(); - Histogram schedulerAllocateHistogram = new Histogram( - new SlidingWindowReservoir(SAMPLING_SIZE)); - metrics.register("sampler.scheduler.operation.allocate.timecost", - schedulerAllocateHistogram); - schedulerHistogramList.add(schedulerAllocateHistogram); - histogramTimerMap.put(schedulerAllocateHistogram, schedulerAllocateTimer); - Histogram schedulerHandleHistogram = new Histogram( - new SlidingWindowReservoir(SAMPLING_SIZE)); - metrics.register("sampler.scheduler.operation.handle.timecost", - schedulerHandleHistogram); - schedulerHistogramList.add(schedulerHandleHistogram); - histogramTimerMap.put(schedulerHandleHistogram, schedulerHandleTimer); - for (SchedulerEventType e : SchedulerEventType.values()) { - Histogram histogram = new Histogram( - new SlidingWindowReservoir(SAMPLING_SIZE)); - metrics.register( - "sampler.scheduler.operation.handle." + e + ".timecost", - histogram); - schedulerHistogramList.add(histogram); - histogramTimerMap.put(histogram, schedulerHandleTimerMap.get(e)); - } - } finally { - samplerLock.unlock(); - } - } - - private void initMetricsCSVOutput() { - int timeIntervalMS = conf.getInt( - SLSConfiguration.METRICS_RECORD_INTERVAL_MS, - SLSConfiguration.METRICS_RECORD_INTERVAL_MS_DEFAULT); - File dir = new File(metricsOutputDir + "/metrics"); - if(! dir.exists() - && ! dir.mkdirs()) { - LOG.error("Cannot create directory " + dir.getAbsoluteFile()); - } - final CsvReporter reporter = CsvReporter.forRegistry(metrics) - .formatFor(Locale.US) - .convertRatesTo(TimeUnit.SECONDS) - .convertDurationsTo(TimeUnit.MILLISECONDS) - .build(new File(metricsOutputDir + "/metrics")); - reporter.start(timeIntervalMS, TimeUnit.MILLISECONDS); - } - - class HistogramsRunnable implements Runnable { - @Override - public void run() { - samplerLock.lock(); - try { - for (Histogram histogram : schedulerHistogramList) { - Timer timer = histogramTimerMap.get(histogram); - histogram.update((int) timer.getSnapshot().getMean()); - } - } finally { - samplerLock.unlock(); - } - } - } - - class MetricsLogRunnable implements Runnable { - private boolean firstLine = true; - public MetricsLogRunnable() { - try { - metricsLogBW = - new BufferedWriter(new OutputStreamWriter(new FileOutputStream( - metricsOutputDir + "/realtimetrack.json"), "UTF-8")); - metricsLogBW.write("["); - } catch (IOException e) { - e.printStackTrace(); - } - } - - @Override - public void run() { - if(running) { - // all WebApp to get real tracking json - String metrics = web.generateRealTimeTrackingMetrics(); - // output - try { - if(firstLine) { - metricsLogBW.write(metrics + EOL); - firstLine = false; - } else { - metricsLogBW.write("," + metrics + EOL); - } - metricsLogBW.flush(); - } catch (IOException e) { - e.printStackTrace(); - } - } - } - } - - // the following functions are used by AMSimulator - public void addAMRuntime(ApplicationId appId, - long traceStartTimeMS, long traceEndTimeMS, - long simulateStartTimeMS, long simulateEndTimeMS) { - if (metricsON) { - try { - // write job runtime information - StringBuilder sb = new StringBuilder(); - sb.append(appId).append(",").append(traceStartTimeMS).append(",") - .append(traceEndTimeMS).append(",").append(simulateStartTimeMS) - .append(",").append(simulateEndTimeMS); - jobRuntimeLogBW.write(sb.toString() + EOL); - jobRuntimeLogBW.flush(); - } catch (IOException e) { - e.printStackTrace(); - } - } - } - - private void updateQueueMetrics(String queue, - long releasedMemory, int releasedVCores) { - // update queue counters - SortedMap counterMap = metrics.getCounters(); - if (releasedMemory != 0) { - String name = "counter.queue." + queue + ".allocated.memory"; - if (! counterMap.containsKey(name)) { - metrics.counter(name); - counterMap = metrics.getCounters(); - } - counterMap.get(name).inc(-releasedMemory); - } - if (releasedVCores != 0) { - String name = "counter.queue." + queue + ".allocated.cores"; - if (! counterMap.containsKey(name)) { - metrics.counter(name); - counterMap = metrics.getCounters(); - } - counterMap.get(name).inc(-releasedVCores); - } - } - - public void setQueueSet(Set queues) { - this.queueSet = queues; - } - - public Set getQueueSet() { - return this.queueSet; - } - - public void setTrackedAppSet(Set apps) { - this.trackedAppSet = apps; - } - - public Set getTrackedAppSet() { - return this.trackedAppSet; - } - - public MetricRegistry getMetrics() { - return metrics; - } - - public SchedulerMetrics getSchedulerMetrics() { - return schedulerMetrics; - } - - // API open to out classes - public void addTrackedApp(ApplicationId appId, String oldAppId) { - if (metricsON) { - schedulerMetrics.trackApp(appId, oldAppId); - } - } - - public void removeTrackedApp(String oldAppId) { - if (metricsON) { - schedulerMetrics.untrackApp(oldAppId); - } - } - - @Override - public Configuration getConf() { - return conf; - } - - @SuppressWarnings("unchecked") - @Override - public void serviceInit(Configuration conf) throws Exception { - ((AbstractYarnScheduler) - scheduler).init(conf); - super.serviceInit(conf); - initScheduler(conf); - } - - private synchronized void initScheduler(Configuration configuration) throws - IOException { - this.applications = - new ConcurrentHashMap>(); - } - - @SuppressWarnings("unchecked") - @Override - public void serviceStart() throws Exception { - ((AbstractYarnScheduler) - scheduler).start(); - super.serviceStart(); - } - - @SuppressWarnings("unchecked") - @Override - public void serviceStop() throws Exception { - ((AbstractYarnScheduler) - scheduler).stop(); - super.serviceStop(); - } - - @Override - public void setRMContext(RMContext rmContext) { - scheduler.setRMContext(rmContext); - } - - @Override - public void reinitialize(Configuration conf, RMContext rmContext) - throws IOException { - scheduler.reinitialize(conf, rmContext); - } - - @Override - public void recover(RMStateStore.RMState rmState) throws Exception { - scheduler.recover(rmState); - } - - @Override - public QueueInfo getQueueInfo(String s, boolean b, boolean b2) - throws IOException { - return scheduler.getQueueInfo(s, b, b2); - } - - @Override - public List getQueueUserAclInfo() { - return scheduler.getQueueUserAclInfo(); - } - - @Override - public Resource getMinimumResourceCapability() { - return scheduler.getMinimumResourceCapability(); - } - - @Override - public Resource getMaximumResourceCapability() { - return scheduler.getMaximumResourceCapability(); - } - - @Override - public ResourceCalculator getResourceCalculator() { - return scheduler.getResourceCalculator(); - } - - @Override - public int getNumClusterNodes() { - return scheduler.getNumClusterNodes(); - } - - @Override - public SchedulerNodeReport getNodeReport(NodeId nodeId) { - return scheduler.getNodeReport(nodeId); - } - - @Override - public SchedulerAppReport getSchedulerAppInfo( - ApplicationAttemptId attemptId) { - return scheduler.getSchedulerAppInfo(attemptId); - } - - @Override - public QueueMetrics getRootQueueMetrics() { - return scheduler.getRootQueueMetrics(); - } - - @Override - public synchronized boolean checkAccess(UserGroupInformation callerUGI, - QueueACL acl, String queueName) { - return scheduler.checkAccess(callerUGI, acl, queueName); - } - - @Override - public ApplicationResourceUsageReport getAppResourceUsageReport( - ApplicationAttemptId appAttemptId) { - return scheduler.getAppResourceUsageReport(appAttemptId); - } - - @Override - public List getAppsInQueue(String queue) { - return scheduler.getAppsInQueue(queue); - } - - @Override - public RMContainer getRMContainer(ContainerId containerId) { - return null; - } - - @Override - public String moveApplication(ApplicationId appId, String newQueue) - throws YarnException { - return scheduler.moveApplication(appId, newQueue); - } - - @Override - @LimitedPrivate("yarn") - @Unstable - public Resource getClusterResource() { - return super.getClusterResource(); - } - - @Override - public synchronized List getTransferredContainers( - ApplicationAttemptId currentAttempt) { - return new ArrayList(); - } - - @Override - public Map> - getSchedulerApplications() { - return new HashMap>(); - } - - @Override - protected void completedContainerInternal(RMContainer rmContainer, - ContainerStatus containerStatus, RMContainerEventType event) { - // do nothing - } - - @Override - public Priority checkAndGetApplicationPriority(Priority priority, - UserGroupInformation user, String queueName, ApplicationId applicationId) - throws YarnException { - // TODO Dummy implementation. - return Priority.newInstance(0); - } - -} diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java index 6ea2ab0f1e5..7c37465d7b6 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java @@ -17,34 +17,19 @@ */ package org.apache.hadoop.yarn.sls.scheduler; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.OutputStreamWriter; -import java.util.ArrayList; -import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Set; -import java.util.SortedMap; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.ShutdownHookManager; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerExitStatus; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -65,117 +50,63 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptR import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; import org.apache.hadoop.yarn.sls.SLSRunner; import org.apache.hadoop.yarn.sls.conf.SLSConfiguration; -import org.apache.hadoop.yarn.sls.web.SLSWebApp; +import org.apache.hadoop.yarn.sls.utils.SLSUtils; import org.apache.hadoop.yarn.util.resource.Resources; -import org.apache.log4j.Logger; -import com.codahale.metrics.Counter; -import com.codahale.metrics.CsvReporter; -import com.codahale.metrics.Gauge; -import com.codahale.metrics.Histogram; -import com.codahale.metrics.MetricRegistry; -import com.codahale.metrics.SlidingWindowReservoir; import com.codahale.metrics.Timer; @Private @Unstable public class SLSCapacityScheduler extends CapacityScheduler implements SchedulerWrapper,Configurable { - private static final String EOL = System.getProperty("line.separator"); - private static final String QUEUE_COUNTER_PREFIX = "counter.queue."; - private static final int SAMPLING_SIZE = 60; - private ScheduledExecutorService pool; - // counters for scheduler allocate/handle operations - private Counter schedulerAllocateCounter; - private Counter schedulerHandleCounter; - private Map schedulerHandleCounterMap; - // Timers for scheduler allocate/handle operations - private Timer schedulerAllocateTimer; - private Timer schedulerHandleTimer; - private Map schedulerHandleTimerMap; - private List schedulerHistogramList; - private Map histogramTimerMap; - private Lock samplerLock; - private Lock queueLock; private Configuration conf; private Map appQueueMap = new ConcurrentHashMap(); - private BufferedWriter jobRuntimeLogBW; - - // Priority of the ResourceSchedulerWrapper shutdown hook. - public static final int SHUTDOWN_HOOK_PRIORITY = 30; - - // web app - private SLSWebApp web; private Map preemptionContainerMap = new ConcurrentHashMap(); // metrics - private MetricRegistry metrics; private SchedulerMetrics schedulerMetrics; private boolean metricsON; - private String metricsOutputDir; - private BufferedWriter metricsLogBW; - private boolean running = false; - private static Map defaultSchedulerMetricsMap = - new HashMap(); - static { - defaultSchedulerMetricsMap.put(FairScheduler.class, - FairSchedulerMetrics.class); - defaultSchedulerMetricsMap.put(FifoScheduler.class, - FifoSchedulerMetrics.class); - defaultSchedulerMetricsMap.put(CapacityScheduler.class, - CapacitySchedulerMetrics.class); - } - // must set by outside - private Set queueSet; - private Set trackedAppSet; + private Tracker tracker; - public final Logger LOG = Logger.getLogger(SLSCapacityScheduler.class); + public Tracker getTracker() { + return tracker; + } public SLSCapacityScheduler() { - samplerLock = new ReentrantLock(); - queueLock = new ReentrantLock(); + tracker = new Tracker(); } @Override public void setConf(Configuration conf) { this.conf = conf; super.setConf(conf); - // start metrics metricsON = conf.getBoolean(SLSConfiguration.METRICS_SWITCH, true); if (metricsON) { try { - initMetrics(); + schedulerMetrics = SchedulerMetrics.getInstance(conf, + CapacityScheduler.class); + schedulerMetrics.init(this, conf); } catch (Exception e) { e.printStackTrace(); } - } - ShutdownHookManager.get().addShutdownHook(new Runnable() { - @Override - public void run() { - try { - if (metricsLogBW != null) { - metricsLogBW.write("]"); - metricsLogBW.close(); + ShutdownHookManager.get().addShutdownHook(new Runnable() { + @Override public void run() { + try { + schedulerMetrics.tearDown(); + } catch (Exception e) { + e.printStackTrace(); } - if (web != null) { - web.stop(); - } - tearDown(); - } catch (Exception e) { - e.printStackTrace(); } - } - }, SHUTDOWN_HOOK_PRIORITY); + }, SLSUtils.SHUTDOWN_HOOK_PRIORITY); + } } @Override @@ -184,7 +115,8 @@ public class SLSCapacityScheduler extends CapacityScheduler implements List strings, List strings2, ContainerUpdates updateRequests) { if (metricsON) { - final Timer.Context context = schedulerAllocateTimer.time(); + final Timer.Context context = schedulerMetrics.getSchedulerAllocateTimer() + .time(); Allocation allocation = null; try { allocation = super @@ -193,7 +125,7 @@ public class SLSCapacityScheduler extends CapacityScheduler implements return allocation; } finally { context.stop(); - schedulerAllocateCounter.inc(); + schedulerMetrics.increaseSchedulerAllocationCounter(); try { updateQueueWithAllocateRequest(allocation, attemptId, resourceRequests, containerIds); @@ -209,74 +141,76 @@ public class SLSCapacityScheduler extends CapacityScheduler implements @Override public void handle(SchedulerEvent schedulerEvent) { - // metrics off - if (! metricsON) { - super.handle(schedulerEvent); - return; - } - if(!running) running = true; + if (!metricsON) { + super.handle(schedulerEvent); + return; + } - // metrics on - Timer.Context handlerTimer = null; - Timer.Context operationTimer = null; + if (!schedulerMetrics.isRunning()) { + schedulerMetrics.setRunning(true); + } - NodeUpdateSchedulerEventWrapper eventWrapper; - try { - //if (schedulerEvent instanceof NodeUpdateSchedulerEvent) { - if (schedulerEvent.getType() == SchedulerEventType.NODE_UPDATE - && schedulerEvent instanceof NodeUpdateSchedulerEvent) { - eventWrapper = new NodeUpdateSchedulerEventWrapper( - (NodeUpdateSchedulerEvent)schedulerEvent); - schedulerEvent = eventWrapper; - updateQueueWithNodeUpdate(eventWrapper); - } else if (schedulerEvent.getType() == SchedulerEventType.APP_ATTEMPT_REMOVED - && schedulerEvent instanceof AppAttemptRemovedSchedulerEvent) { - // check if having AM Container, update resource usage information - AppAttemptRemovedSchedulerEvent appRemoveEvent = - (AppAttemptRemovedSchedulerEvent) schedulerEvent; - ApplicationAttemptId appAttemptId = - appRemoveEvent.getApplicationAttemptID(); - String queue = appQueueMap.get(appAttemptId); - SchedulerAppReport app = super.getSchedulerAppInfo(appAttemptId); - if (! app.getLiveContainers().isEmpty()) { // have 0 or 1 - // should have one container which is AM container - RMContainer rmc = app.getLiveContainers().iterator().next(); - updateQueueMetrics(queue, - rmc.getContainer().getResource().getMemorySize(), - rmc.getContainer().getResource().getVirtualCores()); - } - } + Timer.Context handlerTimer = null; + Timer.Context operationTimer = null; - handlerTimer = schedulerHandleTimer.time(); - operationTimer = schedulerHandleTimerMap - .get(schedulerEvent.getType()).time(); + NodeUpdateSchedulerEventWrapper eventWrapper; + try { + if (schedulerEvent.getType() == SchedulerEventType.NODE_UPDATE + && schedulerEvent instanceof NodeUpdateSchedulerEvent) { + eventWrapper = new NodeUpdateSchedulerEventWrapper( + (NodeUpdateSchedulerEvent)schedulerEvent); + schedulerEvent = eventWrapper; + updateQueueWithNodeUpdate(eventWrapper); + } else if (schedulerEvent.getType() == + SchedulerEventType.APP_ATTEMPT_REMOVED + && schedulerEvent instanceof AppAttemptRemovedSchedulerEvent) { + // check if having AM Container, update resource usage information + AppAttemptRemovedSchedulerEvent appRemoveEvent = + (AppAttemptRemovedSchedulerEvent) schedulerEvent; + ApplicationAttemptId appAttemptId = + appRemoveEvent.getApplicationAttemptID(); + String queue = appQueueMap.get(appAttemptId); + SchedulerAppReport app = super.getSchedulerAppInfo(appAttemptId); + if (!app.getLiveContainers().isEmpty()) { // have 0 or 1 + // should have one container which is AM container + RMContainer rmc = app.getLiveContainers().iterator().next(); + schedulerMetrics.updateQueueMetricsByRelease( + rmc.getContainer().getResource(), queue); + } + } - super.handle(schedulerEvent); - } finally { - if (handlerTimer != null) handlerTimer.stop(); - if (operationTimer != null) operationTimer.stop(); - schedulerHandleCounter.inc(); - schedulerHandleCounterMap.get(schedulerEvent.getType()).inc(); + handlerTimer = schedulerMetrics.getSchedulerHandleTimer().time(); + operationTimer = schedulerMetrics.getSchedulerHandleTimer( + schedulerEvent.getType()).time(); - if (schedulerEvent.getType() == SchedulerEventType.APP_ATTEMPT_REMOVED - && schedulerEvent instanceof AppAttemptRemovedSchedulerEvent) { - SLSRunner.decreaseRemainingApps(); - AppAttemptRemovedSchedulerEvent appRemoveEvent = - (AppAttemptRemovedSchedulerEvent) schedulerEvent; - ApplicationAttemptId appAttemptId = - appRemoveEvent.getApplicationAttemptID(); - appQueueMap.remove(appRemoveEvent.getApplicationAttemptID()); - } else if (schedulerEvent.getType() == SchedulerEventType.APP_ATTEMPT_ADDED - && schedulerEvent instanceof AppAttemptAddedSchedulerEvent) { - AppAttemptAddedSchedulerEvent appAddEvent = - (AppAttemptAddedSchedulerEvent) schedulerEvent; - SchedulerApplication app = - applications.get(appAddEvent.getApplicationAttemptId() + super.handle(schedulerEvent); + } finally { + if (handlerTimer != null) { + handlerTimer.stop(); + } + if (operationTimer != null) { + operationTimer.stop(); + } + schedulerMetrics.increaseSchedulerHandleCounter(schedulerEvent.getType()); + + if (schedulerEvent.getType() == SchedulerEventType.APP_ATTEMPT_REMOVED + && schedulerEvent instanceof AppAttemptRemovedSchedulerEvent) { + SLSRunner.decreaseRemainingApps(); + AppAttemptRemovedSchedulerEvent appRemoveEvent = + (AppAttemptRemovedSchedulerEvent) schedulerEvent; + appQueueMap.remove(appRemoveEvent.getApplicationAttemptID()); + } else if (schedulerEvent.getType() == + SchedulerEventType.APP_ATTEMPT_ADDED + && schedulerEvent instanceof AppAttemptAddedSchedulerEvent) { + AppAttemptAddedSchedulerEvent appAddEvent = + (AppAttemptAddedSchedulerEvent) schedulerEvent; + SchedulerApplication app = + applications.get(appAddEvent.getApplicationAttemptId() .getApplicationId()); - appQueueMap.put(appAddEvent.getApplicationAttemptId(), app.getQueue() - .getQueueName()); - } - } + appQueueMap.put(appAddEvent.getApplicationAttemptId(), app.getQueue() + .getQueueName()); + } + } } private void updateQueueWithNodeUpdate( @@ -316,7 +250,8 @@ public class SLSCapacityScheduler extends CapacityScheduler implements } } // update queue counters - updateQueueMetrics(queue, releasedMemory, releasedVCores); + schedulerMetrics.updateQueueMetricsByRelease( + Resource.newInstance(releasedMemory, releasedVCores), queue); } } } @@ -395,410 +330,13 @@ public class SLSCapacityScheduler extends CapacityScheduler implements } // update metrics - SortedMap counterMap = metrics.getCounters(); - String names[] = new String[]{ - "counter.queue." + queueName + ".pending.memory", - "counter.queue." + queueName + ".pending.cores", - "counter.queue." + queueName + ".allocated.memory", - "counter.queue." + queueName + ".allocated.cores"}; - long values[] = new long[]{pendingResource.getMemorySize(), - pendingResource.getVirtualCores(), - allocatedResource.getMemorySize(), allocatedResource.getVirtualCores()}; - for (int i = names.length - 1; i >= 0; i --) { - if (! counterMap.containsKey(names[i])) { - metrics.counter(names[i]); - counterMap = metrics.getCounters(); - } - counterMap.get(names[i]).inc(values[i]); - } - - queueLock.lock(); - try { - if (! schedulerMetrics.isTracked(queueName)) { - schedulerMetrics.trackQueue(queueName); - } - } finally { - queueLock.unlock(); - } - } - - private void tearDown() throws IOException { - // close job runtime writer - if (jobRuntimeLogBW != null) { - jobRuntimeLogBW.close(); - } - // shut pool - if (pool != null) pool.shutdown(); - } - - @SuppressWarnings({ "unchecked", "rawtypes" }) - private void initMetrics() throws Exception { - metrics = new MetricRegistry(); - // configuration - metricsOutputDir = conf.get(SLSConfiguration.METRICS_OUTPUT_DIR); - int metricsWebAddressPort = conf.getInt( - SLSConfiguration.METRICS_WEB_ADDRESS_PORT, - SLSConfiguration.METRICS_WEB_ADDRESS_PORT_DEFAULT); - // create SchedulerMetrics for current scheduler - String schedulerMetricsType = conf.get(CapacityScheduler.class.getName()); - Class schedulerMetricsClass = schedulerMetricsType == null? - defaultSchedulerMetricsMap.get(CapacityScheduler.class) : - Class.forName(schedulerMetricsType); - schedulerMetrics = (SchedulerMetrics)ReflectionUtils - .newInstance(schedulerMetricsClass, new Configuration()); - schedulerMetrics.init(this, metrics); - - // register various metrics - registerJvmMetrics(); - registerClusterResourceMetrics(); - registerContainerAppNumMetrics(); - registerSchedulerMetrics(); - - // .csv output - initMetricsCSVOutput(); - - // start web app to provide real-time tracking - web = new SLSWebApp(this, metricsWebAddressPort); - web.start(); - - // a thread to update histogram timer - pool = new ScheduledThreadPoolExecutor(2); - pool.scheduleAtFixedRate(new HistogramsRunnable(), 0, 1000, - TimeUnit.MILLISECONDS); - - // a thread to output metrics for real-tiem tracking - pool.scheduleAtFixedRate(new MetricsLogRunnable(), 0, 1000, - TimeUnit.MILLISECONDS); - - // application running information - jobRuntimeLogBW = - new BufferedWriter(new OutputStreamWriter(new FileOutputStream( - metricsOutputDir + "/jobruntime.csv"), "UTF-8")); - jobRuntimeLogBW.write("JobID,real_start_time,real_end_time," + - "simulate_start_time,simulate_end_time" + EOL); - jobRuntimeLogBW.flush(); - } - - private void registerJvmMetrics() { - // add JVM gauges - metrics.register("variable.jvm.free.memory", - new Gauge() { - @Override - public Long getValue() { - return Runtime.getRuntime().freeMemory(); - } - } - ); - metrics.register("variable.jvm.max.memory", - new Gauge() { - @Override - public Long getValue() { - return Runtime.getRuntime().maxMemory(); - } - } - ); - metrics.register("variable.jvm.total.memory", - new Gauge() { - @Override - public Long getValue() { - return Runtime.getRuntime().totalMemory(); - } - } - ); - } - - private void registerClusterResourceMetrics() { - metrics.register("variable.cluster.allocated.memory", - new Gauge() { - @Override - public Long getValue() { - if( getRootQueueMetrics() == null) { - return 0L; - } else { - return getRootQueueMetrics().getAllocatedMB(); - } - } - } - ); - metrics.register("variable.cluster.allocated.vcores", - new Gauge() { - @Override - public Integer getValue() { - if(getRootQueueMetrics() == null) { - return 0; - } else { - return getRootQueueMetrics().getAllocatedVirtualCores(); - } - } - } - ); - metrics.register("variable.cluster.available.memory", - new Gauge() { - @Override - public Long getValue() { - if(getRootQueueMetrics() == null) { - return 0L; - } else { - return getRootQueueMetrics().getAvailableMB(); - } - } - } - ); - metrics.register("variable.cluster.available.vcores", - new Gauge() { - @Override - public Integer getValue() { - if(getRootQueueMetrics() == null) { - return 0; - } else { - return getRootQueueMetrics().getAvailableVirtualCores(); - } - } - } - ); - metrics.register("variable.cluster.reserved.memory", - new Gauge() { - @Override - public Long getValue() { - if(getRootQueueMetrics() == null) { - return 0L; - } else { - return getRootQueueMetrics().getReservedMB(); - } - } - } - ); - metrics.register("variable.cluster.reserved.vcores", - new Gauge() { - @Override - public Integer getValue() { - if(getRootQueueMetrics() == null) { - return 0; - } else { - return getRootQueueMetrics().getReservedVirtualCores(); - } - } - } - ); - } - - private void registerContainerAppNumMetrics() { - metrics.register("variable.running.application", - new Gauge() { - @Override - public Integer getValue() { - if(getRootQueueMetrics() == null) { - return 0; - } else { - return getRootQueueMetrics().getAppsRunning(); - } - } - } - ); - metrics.register("variable.running.container", - new Gauge() { - @Override - public Integer getValue() { - if(getRootQueueMetrics() == null) { - return 0; - } else { - return getRootQueueMetrics().getAllocatedContainers(); - } - } - } - ); - } - - private void registerSchedulerMetrics() { - samplerLock.lock(); - try { - // counters for scheduler operations - schedulerAllocateCounter = metrics.counter( - "counter.scheduler.operation.allocate"); - schedulerHandleCounter = metrics.counter( - "counter.scheduler.operation.handle"); - schedulerHandleCounterMap = new HashMap(); - for (SchedulerEventType e : SchedulerEventType.values()) { - Counter counter = metrics.counter( - "counter.scheduler.operation.handle." + e); - schedulerHandleCounterMap.put(e, counter); - } - // timers for scheduler operations - int timeWindowSize = conf.getInt( - SLSConfiguration.METRICS_TIMER_WINDOW_SIZE, - SLSConfiguration.METRICS_TIMER_WINDOW_SIZE_DEFAULT); - schedulerAllocateTimer = new Timer( - new SlidingWindowReservoir(timeWindowSize)); - schedulerHandleTimer = new Timer( - new SlidingWindowReservoir(timeWindowSize)); - schedulerHandleTimerMap = new HashMap(); - for (SchedulerEventType e : SchedulerEventType.values()) { - Timer timer = new Timer(new SlidingWindowReservoir(timeWindowSize)); - schedulerHandleTimerMap.put(e, timer); - } - // histogram for scheduler operations (Samplers) - schedulerHistogramList = new ArrayList(); - histogramTimerMap = new HashMap(); - Histogram schedulerAllocateHistogram = new Histogram( - new SlidingWindowReservoir(SAMPLING_SIZE)); - metrics.register("sampler.scheduler.operation.allocate.timecost", - schedulerAllocateHistogram); - schedulerHistogramList.add(schedulerAllocateHistogram); - histogramTimerMap.put(schedulerAllocateHistogram, schedulerAllocateTimer); - Histogram schedulerHandleHistogram = new Histogram( - new SlidingWindowReservoir(SAMPLING_SIZE)); - metrics.register("sampler.scheduler.operation.handle.timecost", - schedulerHandleHistogram); - schedulerHistogramList.add(schedulerHandleHistogram); - histogramTimerMap.put(schedulerHandleHistogram, schedulerHandleTimer); - for (SchedulerEventType e : SchedulerEventType.values()) { - Histogram histogram = new Histogram( - new SlidingWindowReservoir(SAMPLING_SIZE)); - metrics.register( - "sampler.scheduler.operation.handle." + e + ".timecost", - histogram); - schedulerHistogramList.add(histogram); - histogramTimerMap.put(histogram, schedulerHandleTimerMap.get(e)); - } - } finally { - samplerLock.unlock(); - } - } - - private void initMetricsCSVOutput() { - int timeIntervalMS = conf.getInt( - SLSConfiguration.METRICS_RECORD_INTERVAL_MS, - SLSConfiguration.METRICS_RECORD_INTERVAL_MS_DEFAULT); - File dir = new File(metricsOutputDir + "/metrics"); - if(! dir.exists() - && ! dir.mkdirs()) { - LOG.error("Cannot create directory " + dir.getAbsoluteFile()); - } - final CsvReporter reporter = CsvReporter.forRegistry(metrics) - .formatFor(Locale.US) - .convertRatesTo(TimeUnit.SECONDS) - .convertDurationsTo(TimeUnit.MILLISECONDS) - .build(new File(metricsOutputDir + "/metrics")); - reporter.start(timeIntervalMS, TimeUnit.MILLISECONDS); - } - - class HistogramsRunnable implements Runnable { - @Override - public void run() { - samplerLock.lock(); - try { - for (Histogram histogram : schedulerHistogramList) { - Timer timer = histogramTimerMap.get(histogram); - histogram.update((int) timer.getSnapshot().getMean()); - } - } finally { - samplerLock.unlock(); - } - } - } - - class MetricsLogRunnable implements Runnable { - private boolean firstLine = true; - public MetricsLogRunnable() { - try { - metricsLogBW = - new BufferedWriter(new OutputStreamWriter(new FileOutputStream( - metricsOutputDir + "/realtimetrack.json"), "UTF-8")); - metricsLogBW.write("["); - } catch (IOException e) { - e.printStackTrace(); - } - } - - @Override - public void run() { - if(running) { - // all WebApp to get real tracking json - String metrics = web.generateRealTimeTrackingMetrics(); - // output - try { - if(firstLine) { - metricsLogBW.write(metrics + EOL); - firstLine = false; - } else { - metricsLogBW.write("," + metrics + EOL); - } - metricsLogBW.flush(); - } catch (IOException e) { - e.printStackTrace(); - } - } - } - } - - // the following functions are used by AMSimulator - public void addAMRuntime(ApplicationId appId, - long traceStartTimeMS, long traceEndTimeMS, - long simulateStartTimeMS, long simulateEndTimeMS) { - - if (metricsON) { - try { - // write job runtime information - StringBuilder sb = new StringBuilder(); - sb.append(appId).append(",").append(traceStartTimeMS).append(",") - .append(traceEndTimeMS).append(",").append(simulateStartTimeMS) - .append(",").append(simulateEndTimeMS); - jobRuntimeLogBW.write(sb.toString() + EOL); - jobRuntimeLogBW.flush(); - } catch (IOException e) { - e.printStackTrace(); - } - } - } - - private void updateQueueMetrics(String queue, - long releasedMemory, int releasedVCores) { - // update queue counters - SortedMap counterMap = metrics.getCounters(); - if (releasedMemory != 0) { - String name = "counter.queue." + queue + ".allocated.memory"; - if (! counterMap.containsKey(name)) { - metrics.counter(name); - counterMap = metrics.getCounters(); - } - counterMap.get(name).inc(-releasedMemory); - } - if (releasedVCores != 0) { - String name = "counter.queue." + queue + ".allocated.cores"; - if (! counterMap.containsKey(name)) { - metrics.counter(name); - counterMap = metrics.getCounters(); - } - counterMap.get(name).inc(-releasedVCores); - } + schedulerMetrics.updateQueueMetrics(pendingResource, allocatedResource, + queueName); } private void initQueueMetrics(CSQueue queue) { if (queue instanceof LeafQueue) { - SortedMap counterMap = metrics.getCounters(); - String queueName = queue.getQueueName(); - String[] names = new String[]{ - QUEUE_COUNTER_PREFIX + queueName + ".pending.memory", - QUEUE_COUNTER_PREFIX + queueName + ".pending.cores", - QUEUE_COUNTER_PREFIX + queueName + ".allocated.memory", - QUEUE_COUNTER_PREFIX + queueName + ".allocated.cores" }; - - for (int i = names.length - 1; i >= 0; i--) { - if (!counterMap.containsKey(names[i])) { - metrics.counter(names[i]); - counterMap = metrics.getCounters(); - } - } - - queueLock.lock(); - try { - if (!schedulerMetrics.isTracked(queueName)) { - schedulerMetrics.trackQueue(queueName); - } - } finally { - queueLock.unlock(); - } - + schedulerMetrics.initQueueMetric(queue.getQueueName()); return; } @@ -811,54 +349,17 @@ public class SLSCapacityScheduler extends CapacityScheduler implements public void serviceInit(Configuration configuration) throws Exception { super.serviceInit(configuration); - initQueueMetrics(getRootQueue()); - } - - public void setQueueSet(Set queues) { - this.queueSet = queues; - } - - public Set getQueueSet() { - return this.queueSet; - } - - public void setTrackedAppSet(Set apps) { - this.trackedAppSet = apps; - } - - public Set getTrackedAppSet() { - return this.trackedAppSet; - } - - public MetricRegistry getMetrics() { - return metrics; + if (metricsON) { + initQueueMetrics(getRootQueue()); + } } public SchedulerMetrics getSchedulerMetrics() { return schedulerMetrics; } - // API open to out classes - public void addTrackedApp(ApplicationId appId, - String oldAppId) { - if (metricsON) { - schedulerMetrics.trackApp(appId, oldAppId); - } - } - - public void removeTrackedApp(String oldAppId) { - if (metricsON) { - schedulerMetrics.untrackApp(oldAppId); - } - } - @Override public Configuration getConf() { return conf; } - - - - -} - +} \ No newline at end of file diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java new file mode 100644 index 00000000000..572dacfc55d --- /dev/null +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java @@ -0,0 +1,339 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.sls.scheduler; + +import com.codahale.metrics.Timer; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.ShutdownHookManager; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerExitStatus; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; +import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.sls.SLSRunner; +import org.apache.hadoop.yarn.sls.conf.SLSConfiguration; +import org.apache.hadoop.yarn.sls.utils.SLSUtils; +import org.apache.hadoop.yarn.util.resource.Resources; + +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +@Private +@Unstable +public class SLSFairScheduler extends FairScheduler + implements SchedulerWrapper, Configurable { + private SchedulerMetrics schedulerMetrics; + private boolean metricsON; + private Tracker tracker; + + private Map preemptionContainerMap = + new ConcurrentHashMap<>(); + + public SchedulerMetrics getSchedulerMetrics() { + return schedulerMetrics; + } + + public Tracker getTracker() { + return tracker; + } + + public SLSFairScheduler() { + tracker = new Tracker(); + } + + @Override + public void setConf(Configuration conf) { + super.setConfig(conf); + + metricsON = conf.getBoolean(SLSConfiguration.METRICS_SWITCH, true); + if (metricsON) { + try { + schedulerMetrics = SchedulerMetrics.getInstance(conf, + FairScheduler.class); + schedulerMetrics.init(this, conf); + } catch (Exception e) { + e.printStackTrace(); + } + + ShutdownHookManager.get().addShutdownHook(new Runnable() { + @Override public void run() { + try { + schedulerMetrics.tearDown(); + } catch (Exception e) { + e.printStackTrace(); + } + } + }, SLSUtils.SHUTDOWN_HOOK_PRIORITY); + } + } + + @Override + public Allocation allocate(ApplicationAttemptId attemptId, + List resourceRequests, List containerIds, + List blacklistAdditions, List blacklistRemovals, + ContainerUpdates updateRequests) { + if (metricsON) { + final Timer.Context context = schedulerMetrics.getSchedulerAllocateTimer() + .time(); + Allocation allocation = null; + try { + allocation = super.allocate(attemptId, resourceRequests, containerIds, + blacklistAdditions, blacklistRemovals, updateRequests); + return allocation; + } finally { + context.stop(); + schedulerMetrics.increaseSchedulerAllocationCounter(); + try { + updateQueueWithAllocateRequest(allocation, attemptId, + resourceRequests, containerIds); + } catch (IOException e) { + e.printStackTrace(); + } + } + } else { + return super.allocate(attemptId, resourceRequests, containerIds, + blacklistAdditions, blacklistRemovals, updateRequests); + } + } + + @Override + public void handle(SchedulerEvent schedulerEvent) { + // metrics off + if (!metricsON) { + super.handle(schedulerEvent); + return; + } + + // metrics on + if(!schedulerMetrics.isRunning()) { + schedulerMetrics.setRunning(true); + } + + Timer.Context handlerTimer = null; + Timer.Context operationTimer = null; + + NodeUpdateSchedulerEventWrapper eventWrapper; + try { + if (schedulerEvent.getType() == SchedulerEventType.NODE_UPDATE + && schedulerEvent instanceof NodeUpdateSchedulerEvent) { + eventWrapper = new NodeUpdateSchedulerEventWrapper( + (NodeUpdateSchedulerEvent)schedulerEvent); + schedulerEvent = eventWrapper; + updateQueueWithNodeUpdate(eventWrapper); + } else if ( + schedulerEvent.getType() == SchedulerEventType.APP_ATTEMPT_REMOVED + && schedulerEvent instanceof AppAttemptRemovedSchedulerEvent) { + // check if having AM Container, update resource usage information + AppAttemptRemovedSchedulerEvent appRemoveEvent = + (AppAttemptRemovedSchedulerEvent) schedulerEvent; + ApplicationAttemptId appAttemptId = + appRemoveEvent.getApplicationAttemptID(); + String queueName = getSchedulerApp(appAttemptId).getQueue().getName(); + SchedulerAppReport app = getSchedulerAppInfo(appAttemptId); + if (!app.getLiveContainers().isEmpty()) { // have 0 or 1 + // should have one container which is AM container + RMContainer rmc = app.getLiveContainers().iterator().next(); + schedulerMetrics.updateQueueMetricsByRelease( + rmc.getContainer().getResource(), queueName); + } + } + + handlerTimer = schedulerMetrics.getSchedulerHandleTimer().time(); + operationTimer = schedulerMetrics.getSchedulerHandleTimer( + schedulerEvent.getType()).time(); + + super.handle(schedulerEvent); + } finally { + if (handlerTimer != null) { + handlerTimer.stop(); + } + if (operationTimer != null) { + operationTimer.stop(); + } + schedulerMetrics.increaseSchedulerHandleCounter(schedulerEvent.getType()); + + if (schedulerEvent.getType() == SchedulerEventType.APP_ATTEMPT_REMOVED + && schedulerEvent instanceof AppAttemptRemovedSchedulerEvent) { + SLSRunner.decreaseRemainingApps(); + } + } + } + + private void updateQueueWithNodeUpdate( + NodeUpdateSchedulerEventWrapper eventWrapper) { + RMNodeWrapper node = (RMNodeWrapper) eventWrapper.getRMNode(); + List containerList = node.getContainerUpdates(); + for (UpdatedContainerInfo info : containerList) { + for (ContainerStatus status : info.getCompletedContainers()) { + ContainerId containerId = status.getContainerId(); + SchedulerAppReport app = super.getSchedulerAppInfo( + containerId.getApplicationAttemptId()); + + if (app == null) { + // this happens for the AM container + // The app have already removed when the NM sends the release + // information. + continue; + } + + int releasedMemory = 0, releasedVCores = 0; + if (status.getExitStatus() == ContainerExitStatus.SUCCESS) { + for (RMContainer rmc : app.getLiveContainers()) { + if (rmc.getContainerId() == containerId) { + Resource resource = rmc.getContainer().getResource(); + releasedMemory += resource.getMemorySize(); + releasedVCores += resource.getVirtualCores(); + break; + } + } + } else if (status.getExitStatus() == ContainerExitStatus.ABORTED) { + if (preemptionContainerMap.containsKey(containerId)) { + Resource preResource = preemptionContainerMap.get(containerId); + releasedMemory += preResource.getMemorySize(); + releasedVCores += preResource.getVirtualCores(); + preemptionContainerMap.remove(containerId); + } + } + // update queue counters + String queue = getSchedulerApp(containerId.getApplicationAttemptId()). + getQueueName(); + schedulerMetrics.updateQueueMetricsByRelease( + Resource.newInstance(releasedMemory, releasedVCores), queue); + } + } + } + + private void updateQueueWithAllocateRequest(Allocation allocation, + ApplicationAttemptId attemptId, + List resourceRequests, + List containerIds) throws IOException { + // update queue information + Resource pendingResource = Resources.createResource(0, 0); + Resource allocatedResource = Resources.createResource(0, 0); + // container requested + for (ResourceRequest request : resourceRequests) { + if (request.getResourceName().equals(ResourceRequest.ANY)) { + Resources.addTo(pendingResource, + Resources.multiply(request.getCapability(), + request.getNumContainers())); + } + } + // container allocated + for (Container container : allocation.getContainers()) { + Resources.addTo(allocatedResource, container.getResource()); + Resources.subtractFrom(pendingResource, container.getResource()); + } + // container released from AM + SchedulerAppReport report = super.getSchedulerAppInfo(attemptId); + for (ContainerId containerId : containerIds) { + Container container = null; + for (RMContainer c : report.getLiveContainers()) { + if (c.getContainerId().equals(containerId)) { + container = c.getContainer(); + break; + } + } + if (container != null) { + // released allocated containers + Resources.subtractFrom(allocatedResource, container.getResource()); + } else { + for (RMContainer c : report.getReservedContainers()) { + if (c.getContainerId().equals(containerId)) { + container = c.getContainer(); + break; + } + } + if (container != null) { + // released reserved containers + Resources.subtractFrom(pendingResource, container.getResource()); + } + } + } + // containers released/preemption from scheduler + Set preemptionContainers = new HashSet(); + if (allocation.getContainerPreemptions() != null) { + preemptionContainers.addAll(allocation.getContainerPreemptions()); + } + if (allocation.getStrictContainerPreemptions() != null) { + preemptionContainers.addAll(allocation.getStrictContainerPreemptions()); + } + if (!preemptionContainers.isEmpty()) { + for (ContainerId containerId : preemptionContainers) { + if (!preemptionContainerMap.containsKey(containerId)) { + Container container = null; + for (RMContainer c : report.getLiveContainers()) { + if (c.getContainerId().equals(containerId)) { + container = c.getContainer(); + break; + } + } + if (container != null) { + preemptionContainerMap.put(containerId, container.getResource()); + } + } + + } + } + + // update metrics + String queueName = getSchedulerApp(attemptId).getQueueName(); + schedulerMetrics.updateQueueMetrics(pendingResource, allocatedResource, + queueName); + } + + private void initQueueMetrics(FSQueue queue) { + if (queue instanceof FSLeafQueue) { + schedulerMetrics.initQueueMetric(queue.getQueueName()); + return; + } + + for (FSQueue child : queue.getChildQueues()) { + initQueueMetrics(child); + } + } + + @Override + public void serviceInit(Configuration conf) throws Exception { + super.serviceInit(conf); + if (metricsON) { + initQueueMetrics(getQueueManager().getRootQueue()); + } + } +} + diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java index 8645a697c1d..a8792e81ec5 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java @@ -18,40 +18,170 @@ package org.apache.hadoop.yarn.sls.scheduler; +import java.io.BufferedWriter; +import java.io.IOException; +import java.util.HashMap; import java.util.HashSet; +import java.util.Map; import java.util.Set; +import java.io.File; +import java.io.FileOutputStream; +import java.io.OutputStreamWriter; +import java.util.ArrayList; +import java.util.List; +import java.util.SortedMap; +import java.util.Locale; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; +import java.util.concurrent.locks.Lock; +import com.codahale.metrics.Counter; +import com.codahale.metrics.CsvReporter; +import com.codahale.metrics.Gauge; +import com.codahale.metrics.Histogram; +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.SlidingWindowReservoir; +import com.codahale.metrics.Timer; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; - -import com.codahale.metrics.Gauge; -import com.codahale.metrics.MetricRegistry; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; +import org.apache.hadoop.yarn.sls.conf.SLSConfiguration; +import org.apache.hadoop.yarn.sls.web.SLSWebApp; +import org.apache.log4j.Logger; @Private @Unstable public abstract class SchedulerMetrics { + private static final String EOL = System.getProperty("line.separator"); + private static final int SAMPLING_SIZE = 60; + private static final Logger LOG = Logger.getLogger(SchedulerMetrics.class); + protected ResourceScheduler scheduler; protected Set trackedQueues; protected MetricRegistry metrics; protected Set appTrackedMetrics; protected Set queueTrackedMetrics; - + + private Configuration conf; + private ScheduledExecutorService pool; + private SLSWebApp web; + + // metrics + private String metricsOutputDir; + private BufferedWriter metricsLogBW; + private BufferedWriter jobRuntimeLogBW; + private boolean running = false; + + // counters for scheduler allocate/handle operations + private Counter schedulerAllocateCounter; + private Counter schedulerHandleCounter; + private Map schedulerHandleCounterMap; + + // Timers for scheduler allocate/handle operations + private Timer schedulerAllocateTimer; + private Timer schedulerHandleTimer; + private Map schedulerHandleTimerMap; + private List schedulerHistogramList; + private Map histogramTimerMap; + private Lock samplerLock; + private Lock queueLock; + + static Class getSchedulerMetricsClass(Configuration conf, + Class schedulerClass) throws ClassNotFoundException { + Class metricClass = null; + String schedulerMetricsType = conf.get(schedulerClass.getName()); + if (schedulerMetricsType != null) { + metricClass = Class.forName(schedulerMetricsType); + } + + if (schedulerClass.equals(FairScheduler.class)) { + metricClass = FairSchedulerMetrics.class; + } else if (schedulerClass.equals(CapacityScheduler.class)) { + metricClass = CapacitySchedulerMetrics.class; + } else if (schedulerClass.equals(FifoScheduler.class)) { + metricClass = FifoSchedulerMetrics.class; + } + + return metricClass; + } + + static SchedulerMetrics getInstance(Configuration conf, Class schedulerClass) + throws ClassNotFoundException { + Class schedulerMetricClass = getSchedulerMetricsClass(conf, schedulerClass); + return (SchedulerMetrics) ReflectionUtils + .newInstance(schedulerMetricClass, new Configuration()); + } + public SchedulerMetrics() { + metrics = new MetricRegistry(); + appTrackedMetrics = new HashSet<>(); appTrackedMetrics.add("live.containers"); appTrackedMetrics.add("reserved.containers"); + queueTrackedMetrics = new HashSet<>(); + trackedQueues = new HashSet<>(); + + samplerLock = new ReentrantLock(); + queueLock = new ReentrantLock(); } - - public void init(ResourceScheduler scheduler, MetricRegistry metrics) { - this.scheduler = scheduler; - this.trackedQueues = new HashSet<>(); - this.metrics = metrics; + + void init(ResourceScheduler resourceScheduler, Configuration config) + throws Exception { + this.scheduler = resourceScheduler; + this.conf = config; + + metricsOutputDir = conf.get(SLSConfiguration.METRICS_OUTPUT_DIR); + + // register various metrics + registerJvmMetrics(); + registerClusterResourceMetrics(); + registerContainerAppNumMetrics(); + registerSchedulerMetrics(); + + // .csv output + initMetricsCSVOutput(); + + // start web app to provide real-time tracking + int metricsWebAddressPort = conf.getInt( + SLSConfiguration.METRICS_WEB_ADDRESS_PORT, + SLSConfiguration.METRICS_WEB_ADDRESS_PORT_DEFAULT); + web = new SLSWebApp((SchedulerWrapper)scheduler, metricsWebAddressPort); + web.start(); + + // a thread to update histogram timer + pool = new ScheduledThreadPoolExecutor(2); + pool.scheduleAtFixedRate(new HistogramsRunnable(), 0, 1000, + TimeUnit.MILLISECONDS); + + // a thread to output metrics for real-tiem tracking + pool.scheduleAtFixedRate(new MetricsLogRunnable(), 0, 1000, + TimeUnit.MILLISECONDS); + + // application running information + jobRuntimeLogBW = + new BufferedWriter(new OutputStreamWriter(new FileOutputStream( + metricsOutputDir + "/jobruntime.csv"), "UTF-8")); + jobRuntimeLogBW.write("JobID,real_start_time,real_end_time," + + "simulate_start_time,simulate_end_time" + EOL); + jobRuntimeLogBW.flush(); + } + + public MetricRegistry getMetrics() { + return metrics; } protected SchedulerApplicationAttempt getSchedulerAppAttempt( @@ -117,7 +247,392 @@ public abstract class SchedulerMetrics { public Set getAppTrackedMetrics() { return appTrackedMetrics; } + public Set getQueueTrackedMetrics() { return queueTrackedMetrics; } + + private void registerJvmMetrics() { + // add JVM gauges + metrics.register("variable.jvm.free.memory", + new Gauge() { + @Override + public Long getValue() { + return Runtime.getRuntime().freeMemory(); + } + } + ); + metrics.register("variable.jvm.max.memory", + new Gauge() { + @Override + public Long getValue() { + return Runtime.getRuntime().maxMemory(); + } + } + ); + metrics.register("variable.jvm.total.memory", + new Gauge() { + @Override + public Long getValue() { + return Runtime.getRuntime().totalMemory(); + } + } + ); + } + + private void registerClusterResourceMetrics() { + metrics.register("variable.cluster.allocated.memory", + new Gauge() { + @Override + public Long getValue() { + if (scheduler.getRootQueueMetrics() == null) { + return 0L; + } else { + return scheduler.getRootQueueMetrics().getAllocatedMB(); + } + } + } + ); + metrics.register("variable.cluster.allocated.vcores", + new Gauge() { + @Override + public Integer getValue() { + if (scheduler.getRootQueueMetrics() == null) { + return 0; + } else { + return scheduler.getRootQueueMetrics().getAllocatedVirtualCores(); + } + } + } + ); + metrics.register("variable.cluster.available.memory", + new Gauge() { + @Override + public Long getValue() { + if (scheduler.getRootQueueMetrics() == null) { + return 0L; + } else { + return scheduler.getRootQueueMetrics().getAvailableMB(); + } + } + } + ); + metrics.register("variable.cluster.available.vcores", + new Gauge() { + @Override + public Integer getValue() { + if (scheduler.getRootQueueMetrics() == null) { + return 0; + } else { + return scheduler.getRootQueueMetrics().getAvailableVirtualCores(); + } + } + } + ); + } + + private void registerContainerAppNumMetrics() { + metrics.register("variable.running.application", + new Gauge() { + @Override + public Integer getValue() { + if (scheduler.getRootQueueMetrics() == null) { + return 0; + } else { + return scheduler.getRootQueueMetrics().getAppsRunning(); + } + } + } + ); + metrics.register("variable.running.container", + new Gauge() { + @Override + public Integer getValue() { + if (scheduler.getRootQueueMetrics() == null) { + return 0; + } else { + return scheduler.getRootQueueMetrics().getAllocatedContainers(); + } + } + } + ); + } + + private void registerSchedulerMetrics() { + samplerLock.lock(); + try { + // counters for scheduler operations + schedulerAllocateCounter = metrics.counter( + "counter.scheduler.operation.allocate"); + schedulerHandleCounter = metrics.counter( + "counter.scheduler.operation.handle"); + schedulerHandleCounterMap = new HashMap<>(); + for (SchedulerEventType e : SchedulerEventType.values()) { + Counter counter = metrics.counter( + "counter.scheduler.operation.handle." + e); + schedulerHandleCounterMap.put(e, counter); + } + // timers for scheduler operations + int timeWindowSize = conf.getInt( + SLSConfiguration.METRICS_TIMER_WINDOW_SIZE, + SLSConfiguration.METRICS_TIMER_WINDOW_SIZE_DEFAULT); + schedulerAllocateTimer = new Timer( + new SlidingWindowReservoir(timeWindowSize)); + schedulerHandleTimer = new Timer( + new SlidingWindowReservoir(timeWindowSize)); + schedulerHandleTimerMap = new HashMap<>(); + for (SchedulerEventType e : SchedulerEventType.values()) { + Timer timer = new Timer(new SlidingWindowReservoir(timeWindowSize)); + schedulerHandleTimerMap.put(e, timer); + } + // histogram for scheduler operations (Samplers) + schedulerHistogramList = new ArrayList<>(); + histogramTimerMap = new HashMap<>(); + Histogram schedulerAllocateHistogram = new Histogram( + new SlidingWindowReservoir(SAMPLING_SIZE)); + metrics.register("sampler.scheduler.operation.allocate.timecost", + schedulerAllocateHistogram); + schedulerHistogramList.add(schedulerAllocateHistogram); + histogramTimerMap.put(schedulerAllocateHistogram, schedulerAllocateTimer); + Histogram schedulerHandleHistogram = new Histogram( + new SlidingWindowReservoir(SAMPLING_SIZE)); + metrics.register("sampler.scheduler.operation.handle.timecost", + schedulerHandleHistogram); + schedulerHistogramList.add(schedulerHandleHistogram); + histogramTimerMap.put(schedulerHandleHistogram, schedulerHandleTimer); + for (SchedulerEventType e : SchedulerEventType.values()) { + Histogram histogram = new Histogram( + new SlidingWindowReservoir(SAMPLING_SIZE)); + metrics.register( + "sampler.scheduler.operation.handle." + e + ".timecost", + histogram); + schedulerHistogramList.add(histogram); + histogramTimerMap.put(histogram, schedulerHandleTimerMap.get(e)); + } + } finally { + samplerLock.unlock(); + } + } + + private void initMetricsCSVOutput() { + int timeIntervalMS = conf.getInt( + SLSConfiguration.METRICS_RECORD_INTERVAL_MS, + SLSConfiguration.METRICS_RECORD_INTERVAL_MS_DEFAULT); + File dir = new File(metricsOutputDir + "/metrics"); + if(!dir.exists() && !dir.mkdirs()) { + LOG.error("Cannot create directory " + dir.getAbsoluteFile()); + } + final CsvReporter reporter = CsvReporter.forRegistry(metrics) + .formatFor(Locale.US) + .convertRatesTo(TimeUnit.SECONDS) + .convertDurationsTo(TimeUnit.MILLISECONDS) + .build(new File(metricsOutputDir + "/metrics")); + reporter.start(timeIntervalMS, TimeUnit.MILLISECONDS); + } + + boolean isRunning() { + return running; + } + + void setRunning(boolean running) { + this.running = running; + } + + class HistogramsRunnable implements Runnable { + @Override + public void run() { + samplerLock.lock(); + try { + for (Histogram histogram : schedulerHistogramList) { + Timer timer = histogramTimerMap.get(histogram); + histogram.update((int) timer.getSnapshot().getMean()); + } + } finally { + samplerLock.unlock(); + } + } + } + + class MetricsLogRunnable implements Runnable { + private boolean firstLine = true; + + MetricsLogRunnable() { + try { + metricsLogBW = + new BufferedWriter(new OutputStreamWriter(new FileOutputStream( + metricsOutputDir + "/realtimetrack.json"), "UTF-8")); + metricsLogBW.write("["); + } catch (IOException e) { + LOG.info(e.getMessage()); + } + } + + @Override + public void run() { + if(running) { + // all WebApp to get real tracking json + String trackingMetrics = web.generateRealTimeTrackingMetrics(); + // output + try { + if(firstLine) { + metricsLogBW.write(trackingMetrics + EOL); + firstLine = false; + } else { + metricsLogBW.write("," + trackingMetrics + EOL); + } + metricsLogBW.flush(); + } catch (IOException e) { + LOG.info(e.getMessage()); + } + } + } + } + + void tearDown() throws Exception { + if (metricsLogBW != null) { + metricsLogBW.write("]"); + metricsLogBW.close(); + } + + if (web != null) { + web.stop(); + } + + if (jobRuntimeLogBW != null) { + jobRuntimeLogBW.close(); + } + + if (pool != null) { + pool.shutdown(); + } + } + + void increaseSchedulerAllocationCounter() { + schedulerAllocateCounter.inc(); + } + + void increaseSchedulerHandleCounter(SchedulerEventType schedulerEventType) { + schedulerHandleCounter.inc(); + schedulerHandleCounterMap.get(schedulerEventType).inc(); + } + + Timer getSchedulerAllocateTimer() { + return schedulerAllocateTimer; + } + + Timer getSchedulerHandleTimer() { + return schedulerHandleTimer; + } + + Timer getSchedulerHandleTimer(SchedulerEventType schedulerEventType) { + return schedulerHandleTimerMap.get(schedulerEventType); + } + + private enum QueueMetric { + PENDING_MEMORY("pending.memory"), + PENDING_VCORES("pending.cores"), + ALLOCATED_MEMORY("allocated.memory"), + ALLOCATED_VCORES("allocated.cores"); + + private String value; + + QueueMetric(String value) { + this.value = value; + } + } + + private String getQueueMetricName(String queue, QueueMetric metric) { + return "counter.queue." + queue + "." + metric.value; + } + + private void traceQueueIfNotTraced(String queue) { + queueLock.lock(); + try { + if (!isTracked(queue)) { + trackQueue(queue); + } + } finally { + queueLock.unlock(); + } + } + + void initQueueMetric(String queueName){ + SortedMap counterMap = metrics.getCounters(); + + for (QueueMetric queueMetric : QueueMetric.values()) { + String metricName = getQueueMetricName(queueName, queueMetric); + if (!counterMap.containsKey(metricName)) { + metrics.counter(metricName); + counterMap = metrics.getCounters(); + } + } + + traceQueueIfNotTraced(queueName); + } + + void updateQueueMetrics(Resource pendingResource, Resource allocatedResource, + String queueName) { + SortedMap counterMap = metrics.getCounters(); + for(QueueMetric metric : QueueMetric.values()) { + String metricName = getQueueMetricName(queueName, metric); + if (!counterMap.containsKey(metricName)) { + metrics.counter(metricName); + counterMap = metrics.getCounters(); + } + + if (metric == QueueMetric.PENDING_MEMORY) { + counterMap.get(metricName).inc(pendingResource.getMemorySize()); + } else if (metric == QueueMetric.PENDING_VCORES) { + counterMap.get(metricName).inc(pendingResource.getVirtualCores()); + } else if (metric == QueueMetric.ALLOCATED_MEMORY) { + counterMap.get(metricName).inc(allocatedResource.getMemorySize()); + } else if (metric == QueueMetric.ALLOCATED_VCORES){ + counterMap.get(metricName).inc(allocatedResource.getVirtualCores()); + } + } + + traceQueueIfNotTraced(queueName); + } + + void updateQueueMetricsByRelease(Resource releaseResource, String queue) { + SortedMap counterMap = metrics.getCounters(); + String name = getQueueMetricName(queue, QueueMetric.ALLOCATED_MEMORY); + if (!counterMap.containsKey(name)) { + metrics.counter(name); + counterMap = metrics.getCounters(); + } + counterMap.get(name).inc(-releaseResource.getMemorySize()); + + String vcoreMetric = + getQueueMetricName(queue, QueueMetric.ALLOCATED_VCORES); + if (!counterMap.containsKey(vcoreMetric)) { + metrics.counter(vcoreMetric); + counterMap = metrics.getCounters(); + } + counterMap.get(vcoreMetric).inc(-releaseResource.getVirtualCores()); + } + + public void addTrackedApp(ApplicationId appId, + String oldAppId) { + trackApp(appId, oldAppId); + } + + public void removeTrackedApp(String oldAppId) { + untrackApp(oldAppId); + } + + public void addAMRuntime(ApplicationId appId, long traceStartTimeMS, + long traceEndTimeMS, long simulateStartTimeMS, long simulateEndTimeMS) { + try { + // write job runtime information + StringBuilder sb = new StringBuilder(); + sb.append(appId).append(",").append(traceStartTimeMS).append(",") + .append(traceEndTimeMS).append(",").append(simulateStartTimeMS) + .append(",").append(simulateEndTimeMS); + jobRuntimeLogBW.write(sb.toString() + EOL); + jobRuntimeLogBW.flush(); + } catch (IOException e) { + LOG.info(e.getMessage()); + } + } } diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerWrapper.java index 962b137a7a5..406f0508eca 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerWrapper.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerWrapper.java @@ -17,28 +17,12 @@ */ package org.apache.hadoop.yarn.sls.scheduler; -import java.util.Set; - import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.records.ApplicationId; - -import com.codahale.metrics.MetricRegistry; @Private @Unstable public interface SchedulerWrapper { - - MetricRegistry getMetrics(); SchedulerMetrics getSchedulerMetrics(); - Set getQueueSet(); - void setQueueSet(Set queues); - Set getTrackedAppSet(); - void setTrackedAppSet(Set apps); - void addTrackedApp(ApplicationId appId, String oldAppId); - void removeTrackedApp(String oldAppId); - void addAMRuntime(ApplicationId appId, - long traceStartTimeMS, long traceEndTimeMS, - long simulateStartTimeMS, long simulateEndTimeMS); - + Tracker getTracker(); } diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/Tracker.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/Tracker.java new file mode 100644 index 00000000000..42a5c3c894d --- /dev/null +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/Tracker.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.sls.scheduler; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; + +import java.util.Set; + +@Private +@Unstable +public class Tracker { + private Set queueSet; + private Set trackedAppSet; + + public void setQueueSet(Set queues) { + queueSet = queues; + } + + public Set getQueueSet() { + return queueSet; + } + + public void setTrackedAppSet(Set apps) { + trackedAppSet = apps; + } + + public Set getTrackedAppSet() { + return trackedAppSet; + } +} diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java index e5f7cd067b5..085edc00856 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java @@ -43,6 +43,7 @@ import org.apache.hadoop.tools.rumen.LoggedTaskAttempt; @Private @Unstable public class SLSUtils { + public static final int SHUTDOWN_HOOK_PRIORITY = 30; // hostname includes the network path and the host name. for example // "/default-rack/hostFoo" or "/coreSwitchA/TORSwitchB/hostBar". diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java index 33d48466c20..2d2ffc55068 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java @@ -107,12 +107,12 @@ public class SLSWebApp extends HttpServlet { public SLSWebApp(SchedulerWrapper wrapper, int metricsAddressPort) { this.wrapper = wrapper; - metrics = wrapper.getMetrics(); handleOperTimecostHistogramMap = new HashMap(); queueAllocatedMemoryCounterMap = new HashMap(); queueAllocatedVCoresCounterMap = new HashMap(); schedulerMetrics = wrapper.getSchedulerMetrics(); + metrics = schedulerMetrics.getMetrics(); port = metricsAddressPort; } @@ -226,7 +226,7 @@ public class SLSWebApp extends HttpServlet { response.setStatus(HttpServletResponse.SC_OK); // queues {0} - Set queues = wrapper.getQueueSet(); + Set queues = wrapper.getTracker().getQueueSet(); StringBuilder queueInfo = new StringBuilder(); int i = 0; @@ -265,7 +265,7 @@ public class SLSWebApp extends HttpServlet { // tracked queues {0} StringBuilder trackedQueueInfo = new StringBuilder(); - Set trackedQueues = wrapper.getQueueSet(); + Set trackedQueues = wrapper.getTracker().getQueueSet(); for(String queue : trackedQueues) { trackedQueueInfo.append(""); @@ -273,7 +273,7 @@ public class SLSWebApp extends HttpServlet { // tracked apps {1} StringBuilder trackedAppInfo = new StringBuilder(); - Set trackedApps = wrapper.getTrackedAppSet(); + Set trackedApps = wrapper.getTracker().getTrackedAppSet(); for(String job : trackedApps) { trackedAppInfo.append(""); @@ -422,7 +422,7 @@ public class SLSWebApp extends HttpServlet { // allocated resource for each queue Map queueAllocatedMemoryMap = new HashMap(); Map queueAllocatedVCoresMap = new HashMap(); - for (String queue : wrapper.getQueueSet()) { + for (String queue : wrapper.getTracker().getQueueSet()) { // memory String key = "counter.queue." + queue + ".allocated.memory"; if (! queueAllocatedMemoryCounterMap.containsKey(queue) && @@ -462,7 +462,7 @@ public class SLSWebApp extends HttpServlet { .append(",\"cluster.available.memory\":").append(availableMemoryGB) .append(",\"cluster.available.vcores\":").append(availableVCoresGB); - for (String queue : wrapper.getQueueSet()) { + for (String queue : wrapper.getTracker().getQueueSet()) { sb.append(",\"queue.").append(queue).append(".allocated.memory\":") .append(queueAllocatedMemoryMap.get(queue)); sb.append(",\"queue.").append(queue).append(".allocated.vcores\":") diff --git a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/TestAMSimulator.java b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/TestAMSimulator.java index f0d8e6f507b..ca3d1958a3b 100644 --- a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/TestAMSimulator.java +++ b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/TestAMSimulator.java @@ -22,37 +22,56 @@ import org.apache.commons.io.FileUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.sls.conf.SLSConfiguration; -import org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator; -import org.apache.hadoop.yarn.sls.scheduler.FairSchedulerMetrics; -import org.apache.hadoop.yarn.sls.scheduler.SchedulerWrapper; +import org.apache.hadoop.yarn.sls.scheduler.*; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.List; +@RunWith(Parameterized.class) public class TestAMSimulator { private ResourceManager rm; private YarnConfiguration conf; private Path metricOutputDir; + private Class slsScheduler; + private Class scheduler; + + @Parameterized.Parameters + public static Collection params() { + return Arrays.asList(new Object[][] { + {SLSFairScheduler.class, FairScheduler.class}, + {SLSCapacityScheduler.class, CapacityScheduler.class} + }); + } + + public TestAMSimulator(Class slsScheduler, Class scheduler) { + this.slsScheduler = slsScheduler; + this.scheduler = scheduler; + } + @Before public void setup() { createMetricOutputDir(); conf = new YarnConfiguration(); conf.set(SLSConfiguration.METRICS_OUTPUT_DIR, metricOutputDir.toString()); - conf.set(YarnConfiguration.RM_SCHEDULER, - "org.apache.hadoop.yarn.sls.scheduler.ResourceSchedulerWrapper"); - conf.set(SLSConfiguration.RM_SCHEDULER, - "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler"); + conf.set(YarnConfiguration.RM_SCHEDULER, slsScheduler.getName()); + conf.set(SLSConfiguration.RM_SCHEDULER, scheduler.getName()); conf.setBoolean(SLSConfiguration.METRICS_SWITCH, true); rm = new ResourceManager(); rm.init(conf); @@ -76,15 +95,17 @@ public class TestAMSimulator { } private void verifySchedulerMetrics(String appId) { - SchedulerWrapper schedulerWrapper = (SchedulerWrapper) - rm.getResourceScheduler(); - MetricRegistry metricRegistry = schedulerWrapper.getMetrics(); - for (FairSchedulerMetrics.Metric metric : - FairSchedulerMetrics.Metric.values()) { - String key = "variable.app." + appId + "." + metric.getValue() - + ".memory"; - Assert.assertTrue(metricRegistry.getGauges().containsKey(key)); - Assert.assertNotNull(metricRegistry.getGauges().get(key).getValue()); + if (scheduler.equals(FairScheduler.class)) { + SchedulerMetrics schedulerMetrics = ((SchedulerWrapper) + rm.getResourceScheduler()).getSchedulerMetrics(); + MetricRegistry metricRegistry = schedulerMetrics.getMetrics(); + for (FairSchedulerMetrics.Metric metric : + FairSchedulerMetrics.Metric.values()) { + String key = "variable.app." + appId + "." + metric.getValue() + + ".memory"; + Assert.assertTrue(metricRegistry.getGauges().containsKey(key)); + Assert.assertNotNull(metricRegistry.getGauges().get(key).getValue()); + } } } diff --git a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/nodemanager/TestNMSimulator.java b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/nodemanager/TestNMSimulator.java index f9a393298eb..2f10f7dbc2e 100644 --- a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/nodemanager/TestNMSimulator.java +++ b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/nodemanager/TestNMSimulator.java @@ -21,26 +21,50 @@ import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.sls.conf.SLSConfiguration; +import org.apache.hadoop.yarn.sls.scheduler.SLSCapacityScheduler; +import org.apache.hadoop.yarn.sls.scheduler.SLSFairScheduler; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import java.util.Arrays; +import java.util.Collection; + +@RunWith(Parameterized.class) public class TestNMSimulator { private final int GB = 1024; private ResourceManager rm; private YarnConfiguration conf; + private Class slsScheduler; + private Class scheduler; + + @Parameterized.Parameters + public static Collection params() { + return Arrays.asList(new Object[][] { + {SLSFairScheduler.class, FairScheduler.class}, + {SLSCapacityScheduler.class, CapacityScheduler.class} + }); + } + + public TestNMSimulator(Class slsScheduler, Class scheduler) { + this.slsScheduler = slsScheduler; + this.scheduler = scheduler; + } + @Before public void setup() { conf = new YarnConfiguration(); - conf.set(YarnConfiguration.RM_SCHEDULER, - "org.apache.hadoop.yarn.sls.scheduler.ResourceSchedulerWrapper"); - conf.set(SLSConfiguration.RM_SCHEDULER, - "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler"); + conf.set(YarnConfiguration.RM_SCHEDULER, slsScheduler.getName()); + conf.set(SLSConfiguration.RM_SCHEDULER, scheduler.getName()); conf.setBoolean(SLSConfiguration.METRICS_SWITCH, false); rm = new ResourceManager(); rm.init(conf); From ceab00ac62f8057a07b4b936799e6f04271e6e41 Mon Sep 17 00:00:00 2001 From: Ming Ma Date: Wed, 29 Mar 2017 17:41:58 -0700 Subject: [PATCH 160/188] MAPREDUCE-6862. Fragments are not handled correctly by resource limit checking. (Chris Trezzo via mingma) --- .../hadoop/mapreduce/JobResourceUploader.java | 36 ++++++++++++----- .../mapreduce/TestJobResourceUploader.java | 40 ++++++++++++++++--- 2 files changed, 59 insertions(+), 17 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java index 4c48ff48c5a..085c96612d9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java @@ -238,28 +238,42 @@ class JobResourceUploader { Collection dcArchives = conf.getStringCollection(MRJobConfig.CACHE_ARCHIVES); - for (String path : dcFiles) { - explorePath(conf, new Path(path), limitChecker, statCache); + for (String uri : dcFiles) { + explorePath(conf, stringToPath(uri), limitChecker, statCache); } - for (String path : dcArchives) { - explorePath(conf, new Path(path), limitChecker, statCache); + for (String uri : dcArchives) { + explorePath(conf, stringToPath(uri), limitChecker, statCache); } - for (String path : files) { - explorePath(conf, new Path(path), limitChecker, statCache); + for (String uri : files) { + explorePath(conf, stringToPath(uri), limitChecker, statCache); } - for (String path : libjars) { - explorePath(conf, new Path(path), limitChecker, statCache); + for (String uri : libjars) { + explorePath(conf, stringToPath(uri), limitChecker, statCache); } - for (String path : archives) { - explorePath(conf, new Path(path), limitChecker, statCache); + for (String uri : archives) { + explorePath(conf, stringToPath(uri), limitChecker, statCache); } if (jobJar != null) { - explorePath(conf, new Path(jobJar), limitChecker, statCache); + explorePath(conf, stringToPath(jobJar), limitChecker, statCache); + } + } + + /** + * Convert a String to a Path and gracefully remove fragments/queries if they + * exist in the String. + */ + @VisibleForTesting + Path stringToPath(String s) { + try { + URI uri = new URI(s); + return new Path(uri.getScheme(), uri.getAuthority(), uri.getPath()); + } catch (URISyntaxException e) { + throw new IllegalArgumentException(e); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJobResourceUploader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJobResourceUploader.java index 36ea57af21d..8ba50a66b3c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJobResourceUploader.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJobResourceUploader.java @@ -39,6 +39,34 @@ import org.junit.Test; */ public class TestJobResourceUploader { + @Test + public void testStringToPath() throws IOException { + Configuration conf = new Configuration(); + JobResourceUploader uploader = + new JobResourceUploader(FileSystem.getLocal(conf), false); + + Assert.assertEquals("Failed: absolute, no scheme, with fragment", + "/testWithFragment.txt", + uploader.stringToPath("/testWithFragment.txt#fragment.txt").toString()); + + Assert.assertEquals("Failed: absolute, with scheme, with fragment", + "file:/testWithFragment.txt", + uploader.stringToPath("file:///testWithFragment.txt#fragment.txt") + .toString()); + + Assert.assertEquals("Failed: relative, no scheme, with fragment", + "testWithFragment.txt", + uploader.stringToPath("testWithFragment.txt#fragment.txt").toString()); + + Assert.assertEquals("Failed: relative, no scheme, no fragment", + "testWithFragment.txt", + uploader.stringToPath("testWithFragment.txt").toString()); + + Assert.assertEquals("Failed: absolute, with scheme, no fragment", + "file:/testWithFragment.txt", + uploader.stringToPath("file:///testWithFragment.txt").toString()); + } + @Test public void testAllDefaults() throws IOException { ResourceLimitsConf.Builder b = new ResourceLimitsConf.Builder(); @@ -210,17 +238,17 @@ public class TestJobResourceUploader { rlConf.maxSingleResourceMB); conf.set("tmpfiles", - buildPathString("file://tmpFiles", rlConf.numOfTmpFiles)); + buildPathString("file:///tmpFiles", rlConf.numOfTmpFiles)); conf.set("tmpjars", - buildPathString("file://tmpjars", rlConf.numOfTmpLibJars)); + buildPathString("file:///tmpjars", rlConf.numOfTmpLibJars)); conf.set("tmparchives", - buildPathString("file://tmpArchives", rlConf.numOfTmpArchives)); + buildPathString("file:///tmpArchives", rlConf.numOfTmpArchives)); conf.set(MRJobConfig.CACHE_ARCHIVES, - buildPathString("file://cacheArchives", rlConf.numOfDCArchives)); + buildPathString("file:///cacheArchives", rlConf.numOfDCArchives)); conf.set(MRJobConfig.CACHE_FILES, - buildPathString("file://cacheFiles", rlConf.numOfDCFiles)); + buildPathString("file:///cacheFiles", rlConf.numOfDCFiles)); if (rlConf.jobJar) { - conf.setJar("file://jobjar.jar"); + conf.setJar("file:///jobjar.jar"); } return conf; } From de7efd2687ccb74608249abf4c83c84a9345ec0a Mon Sep 17 00:00:00 2001 From: Zhe Zhang Date: Wed, 29 Mar 2017 22:10:55 -0700 Subject: [PATCH 161/188] MAPREDUCE-6873. MR Job Submission Fails if MR framework application path not on defaultFS. Contributed by Erik Krogen. --- .../src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java index 09edd948389..6ade376f01d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java @@ -458,7 +458,7 @@ class JobSubmitter { // resolve any symlinks in the URI path so using a "current" symlink // to point to a specific version shows the specific version // in the distributed cache configuration - FileSystem fs = FileSystem.get(conf); + FileSystem fs = FileSystem.get(uri, conf); Path frameworkPath = fs.makeQualified( new Path(uri.getScheme(), uri.getAuthority(), uri.getPath())); FileContext fc = FileContext.getFileContext(frameworkPath.toUri(), conf); From 0d053eeb3002f348d34cf3aac5b43333af8ff10a Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Thu, 30 Mar 2017 18:11:50 +0900 Subject: [PATCH 162/188] HADOOP-14256. [S3A DOC] Correct the format for "Seoul" example. Contributed by Brahma Reddy Battula. --- .../hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index 3e99656a8ee..82c35885301 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -1024,6 +1024,7 @@ Frankfurt ``` Seoul + ```xml fs.s3a.endpoint From c8bd5fc7a86f9890ceaa37a89491ab650e7e9a64 Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Thu, 30 Mar 2017 10:57:19 -0500 Subject: [PATCH 163/188] MAPREDUCE-6850. Shuffle Handler keep-alive connections are closed from the server side. Contributed by Jonathan Eagles --- .../apache/hadoop/mapred/ShuffleHandler.java | 68 +++++++++++++++++-- .../hadoop/mapred/TestShuffleHandler.java | 31 +++++++++ 2 files changed, 92 insertions(+), 7 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java index 15a1b89f84a..863da7efde9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java @@ -104,6 +104,7 @@ import org.jboss.netty.channel.Channel; import org.jboss.netty.channel.ChannelFactory; import org.jboss.netty.channel.ChannelFuture; import org.jboss.netty.channel.ChannelFutureListener; +import org.jboss.netty.channel.ChannelHandler; import org.jboss.netty.channel.ChannelHandlerContext; import org.jboss.netty.channel.ChannelPipeline; import org.jboss.netty.channel.ChannelPipelineFactory; @@ -126,7 +127,13 @@ import org.jboss.netty.handler.codec.http.HttpResponseStatus; import org.jboss.netty.handler.codec.http.QueryStringDecoder; import org.jboss.netty.handler.ssl.SslHandler; import org.jboss.netty.handler.stream.ChunkedWriteHandler; +import org.jboss.netty.handler.timeout.IdleState; +import org.jboss.netty.handler.timeout.IdleStateAwareChannelHandler; +import org.jboss.netty.handler.timeout.IdleStateEvent; +import org.jboss.netty.handler.timeout.IdleStateHandler; import org.jboss.netty.util.CharsetUtil; +import org.jboss.netty.util.HashedWheelTimer; +import org.jboss.netty.util.Timer; import org.eclipse.jetty.http.HttpHeader; import com.google.common.annotations.VisibleForTesting; @@ -240,6 +247,7 @@ public class ShuffleHandler extends AuxiliaryService { public static final boolean DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED = true; public static final boolean WINDOWS_DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED = false; + private static final String TIMEOUT_HANDLER = "timeout"; /* the maximum number of files a single GET request can open simultaneously during shuffle @@ -249,8 +257,9 @@ public class ShuffleHandler extends AuxiliaryService { public static final int DEFAULT_SHUFFLE_MAX_SESSION_OPEN_FILES = 3; boolean connectionKeepAliveEnabled = false; - int connectionKeepAliveTimeOut; - int mapOutputMetaInfoCacheSize; + private int connectionKeepAliveTimeOut; + private int mapOutputMetaInfoCacheSize; + private Timer timer; @Metrics(about="Shuffle output metrics", context="mapred") static class ShuffleMetrics implements ChannelFutureListener { @@ -293,7 +302,15 @@ public class ShuffleHandler extends AuxiliaryService { int waitCount = this.reduceContext.getMapsToWait().decrementAndGet(); if (waitCount == 0) { metrics.operationComplete(future); - future.getChannel().close(); + // Let the idle timer handler close keep-alive connections + if (reduceContext.getKeepAlive()) { + ChannelPipeline pipeline = future.getChannel().getPipeline(); + TimeoutHandler timeoutHandler = + (TimeoutHandler)pipeline.get(TIMEOUT_HANDLER); + timeoutHandler.setEnabledTimeout(true); + } else { + future.getChannel().close(); + } } else { pipelineFact.getSHUFFLE().sendMap(reduceContext); } @@ -314,11 +331,12 @@ public class ShuffleHandler extends AuxiliaryService { private String user; private Map infoMap; private String jobId; + private final boolean keepAlive; public ReduceContext(List mapIds, int rId, ChannelHandlerContext context, String usr, Map mapOutputInfoMap, - String jobId) { + String jobId, boolean keepAlive) { this.mapIds = mapIds; this.reduceId = rId; @@ -339,6 +357,7 @@ public class ShuffleHandler extends AuxiliaryService { this.user = usr; this.infoMap = mapOutputInfoMap; this.jobId = jobId; + this.keepAlive = keepAlive; } public int getReduceId() { @@ -372,6 +391,10 @@ public class ShuffleHandler extends AuxiliaryService { public AtomicInteger getMapsToWait() { return mapsToWait; } + + public boolean getKeepAlive() { + return keepAlive; + } } ShuffleHandler(MetricsSystem ms) { @@ -508,8 +531,10 @@ public class ShuffleHandler extends AuxiliaryService { secretManager = new JobTokenSecretManager(); recoverState(conf); ServerBootstrap bootstrap = new ServerBootstrap(selector); + // Timer is shared across entire factory and must be released separately + timer = new HashedWheelTimer(); try { - pipelineFact = new HttpPipelineFactory(conf); + pipelineFact = new HttpPipelineFactory(conf, timer); } catch (Exception ex) { throw new RuntimeException(ex); } @@ -549,6 +574,10 @@ public class ShuffleHandler extends AuxiliaryService { if (pipelineFact != null) { pipelineFact.destroy(); } + if (timer != null) { + // Release this shared timer resource + timer.stop(); + } if (stateDb != null) { stateDb.close(); } @@ -755,12 +784,29 @@ public class ShuffleHandler extends AuxiliaryService { } } + static class TimeoutHandler extends IdleStateAwareChannelHandler { + + private boolean enabledTimeout; + + void setEnabledTimeout(boolean enabledTimeout) { + this.enabledTimeout = enabledTimeout; + } + + @Override + public void channelIdle(ChannelHandlerContext ctx, IdleStateEvent e) { + if (e.getState() == IdleState.WRITER_IDLE && enabledTimeout) { + e.getChannel().close(); + } + } + } + class HttpPipelineFactory implements ChannelPipelineFactory { final Shuffle SHUFFLE; private SSLFactory sslFactory; + private final ChannelHandler idleStateHandler; - public HttpPipelineFactory(Configuration conf) throws Exception { + public HttpPipelineFactory(Configuration conf, Timer timer) throws Exception { SHUFFLE = getShuffle(conf); if (conf.getBoolean(MRConfig.SHUFFLE_SSL_ENABLED_KEY, MRConfig.SHUFFLE_SSL_ENABLED_DEFAULT)) { @@ -768,6 +814,7 @@ public class ShuffleHandler extends AuxiliaryService { sslFactory = new SSLFactory(SSLFactory.Mode.SERVER, conf); sslFactory.init(); } + this.idleStateHandler = new IdleStateHandler(timer, 0, connectionKeepAliveTimeOut, 0); } public Shuffle getSHUFFLE() { @@ -791,6 +838,8 @@ public class ShuffleHandler extends AuxiliaryService { pipeline.addLast("encoder", new HttpResponseEncoder()); pipeline.addLast("chunking", new ChunkedWriteHandler()); pipeline.addLast("shuffle", SHUFFLE); + pipeline.addLast("idle", idleStateHandler); + pipeline.addLast(TIMEOUT_HANDLER, new TimeoutHandler()); return pipeline; // TODO factor security manager into pipeline // TODO factor out encode/decode to permit binary shuffle @@ -981,6 +1030,10 @@ public class ShuffleHandler extends AuxiliaryService { Map mapOutputInfoMap = new HashMap(); Channel ch = evt.getChannel(); + ChannelPipeline pipeline = ch.getPipeline(); + TimeoutHandler timeoutHandler = + (TimeoutHandler)pipeline.get(TIMEOUT_HANDLER); + timeoutHandler.setEnabledTimeout(false); String user = userRsrc.get(jobId); try { @@ -995,8 +1048,9 @@ public class ShuffleHandler extends AuxiliaryService { } ch.write(response); //Initialize one ReduceContext object per messageReceived call + boolean keepAlive = keepAliveParam || connectionKeepAliveEnabled; ReduceContext reduceContext = new ReduceContext(mapIds, reduceId, ctx, - user, mapOutputInfoMap, jobId); + user, mapOutputInfoMap, jobId, keepAlive); for (int i = 0; i < Math.min(maxSessionOpenFiles, mapIds.size()); i++) { ChannelFuture nextMap = sendMap(reduceContext); if(nextMap == null) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java index 1e439374b5a..7fb2051cb35 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java @@ -37,6 +37,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.net.HttpURLConnection; import java.net.URL; +import java.net.SocketAddress; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; @@ -79,6 +80,7 @@ import org.apache.hadoop.yarn.server.records.Version; import org.jboss.netty.channel.Channel; import org.jboss.netty.channel.ChannelFuture; import org.jboss.netty.channel.ChannelHandlerContext; +import org.jboss.netty.channel.ChannelPipeline; import org.jboss.netty.channel.socket.SocketChannel; import org.jboss.netty.channel.MessageEvent; import org.jboss.netty.channel.AbstractChannel; @@ -309,6 +311,15 @@ public class TestShuffleHandler { Assert.assertTrue("sendError called when client closed connection", failures.size() == 0); } + static class LastSocketAddress { + SocketAddress lastAddress; + void setAddress(SocketAddress lastAddress) { + this.lastAddress = lastAddress; + } + SocketAddress getSocketAddres() { + return lastAddress; + } + } @Test(timeout = 10000) public void testKeepAlive() throws Exception { @@ -318,6 +329,8 @@ public class TestShuffleHandler { conf.setBoolean(ShuffleHandler.SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED, true); // try setting to -ve keep alive timeout. conf.setInt(ShuffleHandler.SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT, -100); + final LastSocketAddress lastSocketAddress = new LastSocketAddress(); + ShuffleHandler shuffleHandler = new ShuffleHandler() { @Override protected Shuffle getShuffle(final Configuration conf) { @@ -363,6 +376,7 @@ public class TestShuffleHandler { protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx, Channel ch, String user, String mapId, int reduce, MapOutputInfo info) throws IOException { + lastSocketAddress.setAddress(ch.getRemoteAddress()); HttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK); // send a shuffle header and a lot of data down the channel @@ -422,6 +436,9 @@ public class TestShuffleHandler { Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); ShuffleHeader header = new ShuffleHeader(); header.readFields(input); + byte[] buffer = new byte[1024]; + while (input.read(buffer) != -1) {} + SocketAddress firstAddress = lastSocketAddress.getSocketAddres(); input.close(); // For keepAlive via URL @@ -443,6 +460,14 @@ public class TestShuffleHandler { header = new ShuffleHeader(); header.readFields(input); input.close(); + SocketAddress secondAddress = lastSocketAddress.getSocketAddres(); + Assert.assertNotNull("Initial shuffle address should not be null", + firstAddress); + Assert.assertNotNull("Keep-Alive shuffle address should not be null", + secondAddress); + Assert.assertEquals("Initial shuffle address and keep-alive shuffle " + + "address should be the same", firstAddress, secondAddress); + } @Test(timeout = 10000) @@ -1058,14 +1083,20 @@ public class TestShuffleHandler { Mockito.mock(ChannelHandlerContext.class); final MessageEvent mockEvt = Mockito.mock(MessageEvent.class); final Channel mockCh = Mockito.mock(AbstractChannel.class); + final ChannelPipeline mockPipeline = Mockito.mock(ChannelPipeline.class); // Mock HttpRequest and ChannelFuture final HttpRequest mockHttpRequest = createMockHttpRequest(); final ChannelFuture mockFuture = createMockChannelFuture(mockCh, listenerList); + final ShuffleHandler.TimeoutHandler timerHandler = + new ShuffleHandler.TimeoutHandler(); // Mock Netty Channel Context and Channel behavior Mockito.doReturn(mockCh).when(mockCtx).getChannel(); + Mockito.when(mockCh.getPipeline()).thenReturn(mockPipeline); + Mockito.when(mockPipeline.get( + Mockito.any(String.class))).thenReturn(timerHandler); Mockito.when(mockCtx.getChannel()).thenReturn(mockCh); Mockito.doReturn(mockFuture).when(mockCh).write(Mockito.any(Object.class)); Mockito.when(mockCh.write(Object.class)).thenReturn(mockFuture); From 7297e82cb4ab1dff06da48acdb76e3d568f998f2 Mon Sep 17 00:00:00 2001 From: Chris Douglas Date: Thu, 30 Mar 2017 10:14:22 -0700 Subject: [PATCH 164/188] HADOOP-14250. Correct spelling of 'separate' and variants. Contributed by Doris Gu --- .../main/java/org/apache/hadoop/net/NodeBase.java | 2 +- .../java/org/apache/hadoop/util/StringUtils.java | 8 ++++---- .../apache/hadoop/fs/http/server/FSOperations.java | 2 +- .../hadoop/mapred/lib/FieldSelectionMapReduce.java | 2 +- .../hadoop/mapred/lib/KeyFieldBasedComparator.java | 2 +- .../org/apache/hadoop/mapreduce/MRJobConfig.java | 8 +++++++- .../lib/fieldsel/FieldSelectionHelper.java | 7 ++++++- .../lib/fieldsel/FieldSelectionMapper.java | 2 +- .../lib/fieldsel/FieldSelectionReducer.java | 2 +- .../lib/input/KeyValueLineRecordReader.java | 13 +++++++++---- .../mapreduce/lib/output/TextOutputFormat.java | 9 +++++++-- .../lib/partition/KeyFieldBasedComparator.java | 4 ++-- .../lib/partition/KeyFieldBasedPartitioner.java | 2 +- .../apache/hadoop/mapreduce/util/ConfigUtil.java | 8 ++++---- .../org/apache/hadoop/fs/slive/OperationOutput.java | 4 ++-- .../apache/hadoop/mapred/TestFieldSelection.java | 2 +- .../mapred/lib/TestKeyFieldBasedComparator.java | 4 ++-- .../lib/fieldsel/TestMRFieldSelection.java | 2 +- .../partition/TestMRKeyFieldBasedComparator.java | 4 ++-- .../apache/hadoop/yarn/conf/YarnConfiguration.java | 2 +- 20 files changed, 55 insertions(+), 34 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NodeBase.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NodeBase.java index 7494adf74c8..9da9ca2948f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NodeBase.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NodeBase.java @@ -45,7 +45,7 @@ public class NodeBase implements Node { /** Construct a node from its path * @param path - * a concatenation of this node's location, the path seperator, and its name + * a concatenation of this node's location, the path separator, and its name */ public NodeBase(String path) { path = normalize(path); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java index d2b572b707d..e773806c651 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java @@ -372,8 +372,8 @@ public class StringUtils { /** * Returns an arraylist of strings. - * @param str the comma seperated string values - * @return the arraylist of the comma seperated string values + * @param str the comma separated string values + * @return the arraylist of the comma separated string values */ public static String[] getStrings(String str){ String delim = ","; @@ -384,7 +384,7 @@ public class StringUtils { * Returns an arraylist of strings. * @param str the string values * @param delim delimiter to separate the values - * @return the arraylist of the seperated string values + * @return the arraylist of the separated string values */ public static String[] getStrings(String str, String delim){ Collection values = getStringCollection(str, delim); @@ -396,7 +396,7 @@ public class StringUtils { /** * Returns a collection of strings. - * @param str comma seperated string values + * @param str comma separated string values * @return an ArrayList of string values */ public static Collection getStringCollection(String str){ diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java index eec6981da02..5ff42a46d31 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java @@ -358,7 +358,7 @@ public class FSOperations { * Creates a Concat executor. * * @param path target path to concat to. - * @param sources comma seperated absolute paths to use as sources. + * @param sources comma separated absolute paths to use as sources. */ public FSConcat(String path, String[] sources) { this.sources = new Path[sources.length]; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/FieldSelectionMapReduce.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/FieldSelectionMapReduce.java index 03335f90693..7a487850eee 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/FieldSelectionMapReduce.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/FieldSelectionMapReduce.java @@ -159,7 +159,7 @@ public class FieldSelectionMapReduce } public void configure(JobConf job) { - this.fieldSeparator = job.get(FieldSelectionHelper.DATA_FIELD_SEPERATOR, + this.fieldSeparator = job.get(FieldSelectionHelper.DATA_FIELD_SEPARATOR, "\t"); this.mapOutputKeyValueSpec = job.get( FieldSelectionHelper.MAP_OUTPUT_KEY_VALUE_SPEC, "0-:"); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/KeyFieldBasedComparator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/KeyFieldBasedComparator.java index c989d77b20f..7eda92a96df 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/KeyFieldBasedComparator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/KeyFieldBasedComparator.java @@ -37,7 +37,7 @@ import org.apache.hadoop.mapreduce.JobContext; * of the field); if omitted from pos2, it defaults to 0 (the end of the * field). opts are ordering options (any of 'nr' as described above). * We assume that the fields in the key are separated by - * {@link JobContext#MAP_OUTPUT_KEY_FIELD_SEPERATOR} + * {@link JobContext#MAP_OUTPUT_KEY_FIELD_SEPARATOR} */ @InterfaceAudience.Public @InterfaceStability.Stable diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index 0a4d222f916..18bf1391fee 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -318,7 +318,13 @@ public interface MRJobConfig { public static final String MAP_OUTPUT_VALUE_CLASS = "mapreduce.map.output.value.class"; - public static final String MAP_OUTPUT_KEY_FIELD_SEPERATOR = "mapreduce.map.output.key.field.separator"; + public static final String MAP_OUTPUT_KEY_FIELD_SEPARATOR = "mapreduce.map.output.key.field.separator"; + + /** + * @deprecated Use {@link #MAP_OUTPUT_KEY_FIELD_SEPARATOR} + */ + @Deprecated + public static final String MAP_OUTPUT_KEY_FIELD_SEPERATOR = MAP_OUTPUT_KEY_FIELD_SEPARATOR; public static final String MAP_LOG_LEVEL = "mapreduce.map.log.level"; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/fieldsel/FieldSelectionHelper.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/fieldsel/FieldSelectionHelper.java index 6e22fe90442..5ee7e0f78b6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/fieldsel/FieldSelectionHelper.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/fieldsel/FieldSelectionHelper.java @@ -60,8 +60,13 @@ import org.apache.hadoop.io.Text; public class FieldSelectionHelper { public static Text emptyText = new Text(""); - public static final String DATA_FIELD_SEPERATOR = + public static final String DATA_FIELD_SEPARATOR = "mapreduce.fieldsel.data.field.separator"; + /** + * @deprecated Use {@link #DATA_FIELD_SEPARATOR} + */ + @Deprecated + public static final String DATA_FIELD_SEPERATOR = DATA_FIELD_SEPARATOR; public static final String MAP_OUTPUT_KEY_VALUE_SPEC = "mapreduce.fieldsel.map.output.key.value.fields.spec"; public static final String REDUCE_OUTPUT_KEY_VALUE_SPEC = diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/fieldsel/FieldSelectionMapper.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/fieldsel/FieldSelectionMapper.java index 627e6626330..fbc1f606b04 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/fieldsel/FieldSelectionMapper.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/fieldsel/FieldSelectionMapper.java @@ -79,7 +79,7 @@ public class FieldSelectionMapper throws IOException, InterruptedException { Configuration conf = context.getConfiguration(); this.fieldSeparator = - conf.get(FieldSelectionHelper.DATA_FIELD_SEPERATOR, "\t"); + conf.get(FieldSelectionHelper.DATA_FIELD_SEPARATOR, "\t"); this.mapOutputKeyValueSpec = conf.get(FieldSelectionHelper.MAP_OUTPUT_KEY_VALUE_SPEC, "0-:"); try { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/fieldsel/FieldSelectionReducer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/fieldsel/FieldSelectionReducer.java index 8a6df33cbe2..5feeaf4692c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/fieldsel/FieldSelectionReducer.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/fieldsel/FieldSelectionReducer.java @@ -77,7 +77,7 @@ public class FieldSelectionReducer Configuration conf = context.getConfiguration(); this.fieldSeparator = - conf.get(FieldSelectionHelper.DATA_FIELD_SEPERATOR, "\t"); + conf.get(FieldSelectionHelper.DATA_FIELD_SEPARATOR, "\t"); this.reduceOutputKeyValueSpec = conf.get(FieldSelectionHelper.REDUCE_OUTPUT_KEY_VALUE_SPEC, "0-:"); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/KeyValueLineRecordReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/KeyValueLineRecordReader.java index 6d661211cf0..06ab6b3f76f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/KeyValueLineRecordReader.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/KeyValueLineRecordReader.java @@ -37,9 +37,14 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext; @InterfaceAudience.Public @InterfaceStability.Stable public class KeyValueLineRecordReader extends RecordReader { - public static final String KEY_VALUE_SEPERATOR = - "mapreduce.input.keyvaluelinerecordreader.key.value.separator"; - + public static final String KEY_VALUE_SEPARATOR = + "mapreduce.input.keyvaluelinerecordreader.key.value.separator"; + /** + * @deprecated Use {@link #KEY_VALUE_SEPARATOR} + */ + @Deprecated + public static final String KEY_VALUE_SEPERATOR = KEY_VALUE_SEPARATOR; + private final LineRecordReader lineRecordReader; private byte separator = (byte) '\t'; @@ -56,7 +61,7 @@ public class KeyValueLineRecordReader extends RecordReader { throws IOException { lineRecordReader = new LineRecordReader(); - String sepStr = conf.get(KEY_VALUE_SEPERATOR, "\t"); + String sepStr = conf.get(KEY_VALUE_SEPARATOR, "\t"); this.separator = (byte) sepStr.charAt(0); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/TextOutputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/TextOutputFormat.java index 2e49f68edab..8b27223df08 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/TextOutputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/TextOutputFormat.java @@ -42,7 +42,12 @@ import org.apache.hadoop.util.*; @InterfaceAudience.Public @InterfaceStability.Stable public class TextOutputFormat extends FileOutputFormat { - public static String SEPERATOR = "mapreduce.output.textoutputformat.separator"; + public static String SEPARATOR = "mapreduce.output.textoutputformat.separator"; + /** + * @deprecated Use {@link #SEPARATOR} + */ + @Deprecated + public static String SEPERATOR = SEPARATOR; protected static class LineRecordWriter extends RecordWriter { private static final byte[] NEWLINE = @@ -107,7 +112,7 @@ public class TextOutputFormat extends FileOutputFormat { ) throws IOException, InterruptedException { Configuration conf = job.getConfiguration(); boolean isCompressed = getCompressOutput(job); - String keyValueSeparator= conf.get(SEPERATOR, "\t"); + String keyValueSeparator= conf.get(SEPARATOR, "\t"); CompressionCodec codec = null; String extension = ""; if (isCompressed) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/KeyFieldBasedComparator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/KeyFieldBasedComparator.java index 4b4d0e9e08e..416bc8c4764 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/KeyFieldBasedComparator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/KeyFieldBasedComparator.java @@ -46,7 +46,7 @@ import org.apache.hadoop.mapreduce.lib.partition.KeyFieldHelper.KeyDescription; * of the field); if omitted from pos2, it defaults to 0 (the end of the * field). opts are ordering options (any of 'nr' as described above). * We assume that the fields in the key are separated by - * {@link JobContext#MAP_OUTPUT_KEY_FIELD_SEPERATOR}. + * {@link JobContext#MAP_OUTPUT_KEY_FIELD_SEPARATOR}. */ @InterfaceAudience.Public @InterfaceStability.Stable @@ -62,7 +62,7 @@ public class KeyFieldBasedComparator extends WritableComparator public void setConf(Configuration conf) { this.conf = conf; String option = conf.get(COMPARATOR_OPTIONS); - String keyFieldSeparator = conf.get(MRJobConfig.MAP_OUTPUT_KEY_FIELD_SEPERATOR,"\t"); + String keyFieldSeparator = conf.get(MRJobConfig.MAP_OUTPUT_KEY_FIELD_SEPARATOR,"\t"); keyFieldHelper.setKeyFieldSeparator(keyFieldSeparator); keyFieldHelper.parseOption(option); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/KeyFieldBasedPartitioner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/KeyFieldBasedPartitioner.java index 44cb624c5a8..73f6a344ce4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/KeyFieldBasedPartitioner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/partition/KeyFieldBasedPartitioner.java @@ -65,7 +65,7 @@ public class KeyFieldBasedPartitioner extends Partitioner this.conf = conf; keyFieldHelper = new KeyFieldHelper(); String keyFieldSeparator = - conf.get(MRJobConfig.MAP_OUTPUT_KEY_FIELD_SEPERATOR, "\t"); + conf.get(MRJobConfig.MAP_OUTPUT_KEY_FIELD_SEPARATOR, "\t"); keyFieldHelper.setKeyFieldSeparator(keyFieldSeparator); if (conf.get("num.key.fields.for.partition") != null) { LOG.warn("Using deprecated num.key.fields.for.partition. " + diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java index d3f62e4c930..4b473792717 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java @@ -290,7 +290,7 @@ public class ConfigUtil { new DeprecationDelta("mapred.mapoutput.value.class", MRJobConfig.MAP_OUTPUT_VALUE_CLASS), new DeprecationDelta("map.output.key.field.separator", - MRJobConfig.MAP_OUTPUT_KEY_FIELD_SEPERATOR), + MRJobConfig.MAP_OUTPUT_KEY_FIELD_SEPARATOR), new DeprecationDelta("mapred.map.child.log.level", MRJobConfig.MAP_LOG_LEVEL), new DeprecationDelta("mapred.inmem.merge.threshold", @@ -412,7 +412,7 @@ public class ConfigUtil { ControlledJob.CREATE_DIR), new DeprecationDelta("mapred.data.field.separator", org.apache.hadoop.mapreduce.lib.fieldsel. - FieldSelectionHelper.DATA_FIELD_SEPERATOR), + FieldSelectionHelper.DATA_FIELD_SEPARATOR), new DeprecationDelta("map.output.key.value.fields.spec", org.apache.hadoop.mapreduce.lib.fieldsel. FieldSelectionHelper.MAP_OUTPUT_KEY_VALUE_SPEC), @@ -427,7 +427,7 @@ public class ConfigUtil { CombineFileInputFormat.SPLIT_MINSIZE_PERRACK), new DeprecationDelta("key.value.separator.in.input.line", org.apache.hadoop.mapreduce.lib.input. - KeyValueLineRecordReader.KEY_VALUE_SEPERATOR), + KeyValueLineRecordReader.KEY_VALUE_SEPARATOR), new DeprecationDelta("mapred.linerecordreader.maxlength", org.apache.hadoop.mapreduce.lib.input. LineRecordReader.MAX_LINE_LENGTH), @@ -436,7 +436,7 @@ public class ConfigUtil { LazyOutputFormat.OUTPUT_FORMAT), new DeprecationDelta("mapred.textoutputformat.separator", org.apache.hadoop.mapreduce.lib.output. - TextOutputFormat.SEPERATOR), + TextOutputFormat.SEPARATOR), new DeprecationDelta("mapred.join.expr", org.apache.hadoop.mapreduce.lib.join. CompositeInputFormat.JOIN_EXPR), diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/OperationOutput.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/OperationOutput.java index bca5a1c777e..02584f19503 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/OperationOutput.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/OperationOutput.java @@ -65,7 +65,7 @@ class OperationOutput { int place = key.indexOf(TYPE_SEP); if (place == -1) { throw new IllegalArgumentException( - "Invalid key format - no type seperator - " + TYPE_SEP); + "Invalid key format - no type separator - " + TYPE_SEP); } try { dataType = OutputType.valueOf( @@ -78,7 +78,7 @@ class OperationOutput { place = key.indexOf(MEASUREMENT_SEP); if (place == -1) { throw new IllegalArgumentException( - "Invalid key format - no measurement seperator - " + MEASUREMENT_SEP); + "Invalid key format - no measurement separator - " + MEASUREMENT_SEP); } opType = key.substring(0, place); measurementType = key.substring(place + 1); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFieldSelection.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFieldSelection.java index 868896815ef..4e14797a161 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFieldSelection.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFieldSelection.java @@ -77,7 +77,7 @@ private static NumberFormat idFormat = NumberFormat.getInstance(); job.setOutputFormat(TextOutputFormat.class); job.setNumReduceTasks(1); - job.set(FieldSelectionHelper.DATA_FIELD_SEPERATOR, "-"); + job.set(FieldSelectionHelper.DATA_FIELD_SEPARATOR, "-"); job.set(FieldSelectionHelper.MAP_OUTPUT_KEY_VALUE_SPEC, "6,5,1-3:0-"); job.set(FieldSelectionHelper.REDUCE_OUTPUT_KEY_VALUE_SPEC, ":4,3,2,1,0,0-"); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestKeyFieldBasedComparator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestKeyFieldBasedComparator.java index 35b3f243c7f..3f31546c789 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestKeyFieldBasedComparator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestKeyFieldBasedComparator.java @@ -63,7 +63,7 @@ public class TestKeyFieldBasedComparator extends HadoopTestCase { super(HadoopTestCase.LOCAL_MR, HadoopTestCase.LOCAL_FS, 1, 1); conf = createJobConf(); localConf = createJobConf(); - localConf.set(JobContext.MAP_OUTPUT_KEY_FIELD_SEPERATOR, " "); + localConf.set(JobContext.MAP_OUTPUT_KEY_FIELD_SEPARATOR, " "); } public void configure(String keySpec, int expect) throws Exception { @@ -85,7 +85,7 @@ public class TestKeyFieldBasedComparator extends HadoopTestCase { conf.setOutputKeyComparatorClass(KeyFieldBasedComparator.class); conf.setKeyFieldComparatorOptions(keySpec); conf.setKeyFieldPartitionerOptions("-k1.1,1.1"); - conf.set(JobContext.MAP_OUTPUT_KEY_FIELD_SEPERATOR, " "); + conf.set(JobContext.MAP_OUTPUT_KEY_FIELD_SEPARATOR, " "); conf.setMapperClass(InverseMapper.class); conf.setReducerClass(IdentityReducer.class); if (!fs.mkdirs(testdir)) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/fieldsel/TestMRFieldSelection.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/fieldsel/TestMRFieldSelection.java index 6f9183ab21b..26cc328bc94 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/fieldsel/TestMRFieldSelection.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/fieldsel/TestMRFieldSelection.java @@ -57,7 +57,7 @@ private static NumberFormat idFormat = NumberFormat.getInstance(); StringBuffer expectedOutput = new StringBuffer(); constructInputOutputData(inputData, expectedOutput, numOfInputLines); - conf.set(FieldSelectionHelper.DATA_FIELD_SEPERATOR, "-"); + conf.set(FieldSelectionHelper.DATA_FIELD_SEPARATOR, "-"); conf.set(FieldSelectionHelper.MAP_OUTPUT_KEY_VALUE_SPEC, "6,5,1-3:0-"); conf.set( FieldSelectionHelper.REDUCE_OUTPUT_KEY_VALUE_SPEC, ":4,3,2,1,0,0-"); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/TestMRKeyFieldBasedComparator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/TestMRKeyFieldBasedComparator.java index 0d75d2fe9ea..889137c20e5 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/TestMRKeyFieldBasedComparator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/partition/TestMRKeyFieldBasedComparator.java @@ -50,7 +50,7 @@ public class TestMRKeyFieldBasedComparator extends HadoopTestCase { public TestMRKeyFieldBasedComparator() throws IOException { super(HadoopTestCase.LOCAL_MR, HadoopTestCase.LOCAL_FS, 1, 1); conf = createJobConf(); - conf.set(MRJobConfig.MAP_OUTPUT_KEY_FIELD_SEPERATOR, " "); + conf.set(MRJobConfig.MAP_OUTPUT_KEY_FIELD_SEPARATOR, " "); } private void testComparator(String keySpec, int expect) @@ -61,7 +61,7 @@ public class TestMRKeyFieldBasedComparator extends HadoopTestCase { conf.set("mapreduce.partition.keycomparator.options", keySpec); conf.set("mapreduce.partition.keypartitioner.options", "-k1.1,1.1"); - conf.set(MRJobConfig.MAP_OUTPUT_KEY_FIELD_SEPERATOR, " "); + conf.set(MRJobConfig.MAP_OUTPUT_KEY_FIELD_SEPARATOR, " "); Job job = MapReduceTestUtil.createJob(conf, inDir, outDir, 1, 1, line1 +"\n" + line2 + "\n"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index f52e487524d..c2392499f70 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1897,7 +1897,7 @@ public class YarnConfiguration extends Configuration { public static final float DEFAULT_TIMELINE_SERVICE_VERSION = 1.0f; /** - * Comma seperated list of names for UIs hosted in the timeline server + * Comma separated list of names for UIs hosted in the timeline server * (For pluggable UIs). */ public static final String TIMELINE_SERVICE_UI_NAMES = From c9b7ce92737f0a829408936ed9a361220d3e9680 Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Thu, 30 Mar 2017 12:54:11 -0500 Subject: [PATCH 165/188] MAPREDUCE-6836. exception thrown when accessing the job configuration web UI. Contributed by Haibo Chen --- .../org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java index 532c2bd4fa4..98a2ce1d0cc 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java @@ -70,7 +70,7 @@ public class ConfBlock extends HtmlBlock { try { ConfInfo info = new ConfInfo(job); - html.div().a("/jobhistory/downloadconf/" + jid, confPath.toString()); + html.div().a("/jobhistory/downloadconf/" + jid, confPath.toString())._(); TBODY> tbody = html. // Tasks table table("#conf"). From 8c591b8d199325f49b5bba29240ca25610cf80a0 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Thu, 30 Mar 2017 11:16:05 -0700 Subject: [PATCH 166/188] HDFS-10974. Document replication factor for EC files. Contributed by Yiqun Lin. --- .../org/apache/hadoop/fs/shell/SetReplication.java | 13 +++++++------ .../src/site/markdown/FileSystemShell.md | 2 +- .../hadoop-common/src/test/resources/testConf.xml | 2 +- .../src/site/markdown/HDFSErasureCoding.md | 1 + .../hadoop-distcp/src/site/markdown/DistCp.md.vm | 2 +- 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java index fab0349a360..2231c58f2b5 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java @@ -41,12 +41,13 @@ class SetReplication extends FsCommand { public static final String NAME = "setrep"; public static final String USAGE = "[-R] [-w] ..."; public static final String DESCRIPTION = - "Set the replication level of a file. If is a directory " + - "then the command recursively changes the replication factor of " + - "all files under the directory tree rooted at .\n" + - "-w: It requests that the command waits for the replication " + - "to complete. This can potentially take a very long time.\n" + - "-R: It is accepted for backwards compatibility. It has no effect."; + "Set the replication level of a file. If is a directory " + + "then the command recursively changes the replication factor of " + + "all files under the directory tree rooted at . " + + "The EC files will be ignored here.\n" + + "-w: It requests that the command waits for the replication " + + "to complete. This can potentially take a very long time.\n" + + "-R: It is accepted for backwards compatibility. It has no effect."; protected short newRep = 0; protected List waitList = new LinkedList(); diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md index 42fddc94fd6..5db96eb4485 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md @@ -647,7 +647,7 @@ setrep Usage: `hadoop fs -setrep [-R] [-w] ` -Changes the replication factor of a file. If *path* is a directory then the command recursively changes the replication factor of all files under the directory tree rooted at *path*. +Changes the replication factor of a file. If *path* is a directory then the command recursively changes the replication factor of all files under the directory tree rooted at *path*. The EC files will be ignored when executing this command. Options: diff --git a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml index 112aea00029..6347aa0d3ea 100644 --- a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml +++ b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml @@ -778,7 +778,7 @@ RegexpComparator - ^\s*rooted at <path>\.( )* + ^\s*rooted at <path>\. The EC files will be ignored here\.( )* RegexpComparator diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md index 04acdce6ea0..f0c487df3d8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md @@ -23,6 +23,7 @@ Purpose However, for warm and cold datasets with relatively low I/O activities, additional block replicas are rarely accessed during normal operations, but still consume the same amount of resources as the first replica. Therefore, a natural improvement is to use Erasure Coding (EC) in place of replication, which provides the same level of fault-tolerance with much less storage space. In typical Erasure Coding (EC) setups, the storage overhead is no more than 50%. + Replication factor of an EC file is meaningless. It is always 1 and cannot be changed via -setrep command. Background ---------- diff --git a/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm b/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm index dbf0e8da669..41a6e94aeee 100644 --- a/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm +++ b/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm @@ -217,7 +217,7 @@ Command Line Options Flag | Description | Notes ----------------- | ------------------------------------ | -------- -`-p[rbugpcaxt]` | Preserve r: replication number b: block size u: user g: group p: permission c: checksum-type a: ACL x: XAttr t: timestamp | When `-update` is specified, status updates will **not** be synchronized unless the file sizes also differ (i.e. unless the file is re-created). If -pa is specified, DistCp preserves the permissions also because ACLs are a super-set of permissions. +`-p[rbugpcaxt]` | Preserve r: replication number b: block size u: user g: group p: permission c: checksum-type a: ACL x: XAttr t: timestamp | When `-update` is specified, status updates will **not** be synchronized unless the file sizes also differ (i.e. unless the file is re-created). If -pa is specified, DistCp preserves the permissions also because ACLs are a super-set of permissions. The option -pr is only valid if both source and target directory are not erasure coded. `-i` | Ignore failures | As explained in the Appendix, this option will keep more accurate statistics about the copy than the default case. It also preserves logs from failed copies, which can be valuable for debugging. Finally, a failing map will not cause the job to fail before all splits are attempted. `-log ` | Write logs to \ | DistCp keeps logs of each file it attempts to copy as map output. If a map fails, the log output will not be retained if it is re-executed. `-m ` | Maximum number of simultaneous copies | Specify the number of maps to copy data. Note that more maps may not necessarily improve throughput. From 1309c585fb9f632f7c649464ecbe358c5130b142 Mon Sep 17 00:00:00 2001 From: Jonathan Eagles Date: Thu, 30 Mar 2017 14:14:43 -0500 Subject: [PATCH 167/188] HADOOP-14216. Addendum to Improve Configuration XML Parsing Performance (jeagles) --- .../main/java/org/apache/hadoop/conf/Configuration.java | 9 --------- 1 file changed, 9 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index 36845e393bd..2ac52cb3949 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -2756,9 +2756,6 @@ public class Configuration implements Iterable>, token.setLength(0); break; case "include": - if (!"xi".equals(reader.getPrefix())) { - break; - } // Determine href for xi:include String confInclude = null; attrCount = reader.getAttributeCount(); @@ -2795,9 +2792,6 @@ public class Configuration implements Iterable>, } break; case "fallback": - if (!"xi".equals(reader.getPrefix())) { - break; - } fallbackEntered = true; break; case "configuration": @@ -2833,9 +2827,6 @@ public class Configuration implements Iterable>, confSource.add(StringInterner.weakIntern(token.toString())); break; case "include": - if (!"xi".equals(reader.getPrefix())) { - break; - } if (fallbackAllowed && !fallbackEntered) { throw new IOException("Fetch fail on include with no " + "fallback while loading '" + name + "'"); From 7c2bc444b3d6750aafeed9b530c8e5b1bf95c1f4 Mon Sep 17 00:00:00 2001 From: Varun Saxena Date: Fri, 31 Mar 2017 02:02:57 +0530 Subject: [PATCH 168/188] YARN-6342. Make TimelineV2Client's drain timeout after stop configurable (Haibo Chen via Varun Saxena) --- .../org/apache/hadoop/yarn/conf/YarnConfiguration.java | 10 ++++++++++ .../yarn/client/api/impl/TimelineV2ClientImpl.java | 9 ++++++--- .../src/main/resources/yarn-default.xml | 9 +++++++++ 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index c2392499f70..81cb8c6c453 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2095,6 +2095,16 @@ public class YarnConfiguration extends Configuration { public static final int DEFAULT_NUMBER_OF_ASYNC_ENTITIES_TO_MERGE = 10; + + /** + * The time period for which timeline v2 client will wait for draining + * leftover entities after stop. + */ + public static final String TIMELINE_V2_CLIENT_DRAIN_TIME_MILLIS = + TIMELINE_SERVICE_CLIENT_PREFIX + "drain-entities.timeout.ms"; + public static final long DEFAULT_TIMELINE_V2_CLIENT_DRAIN_TIME_MILLIS + = 2000L; + // mark app-history related configs @Private as application history is going // to be integrated into the timeline service @Private diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineV2ClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineV2ClientImpl.java index cef7e5f579e..5d88f708b15 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineV2ClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineV2ClientImpl.java @@ -289,7 +289,7 @@ public class TimelineV2ClientImpl extends TimelineV2Client { * Time period for which the timelineclient will wait for draining after * stop. */ - private static final long DRAIN_TIME_PERIOD = 2000L; + private final long drainTimeoutPeriod; private int numberOfAsyncsToMerge; private final BlockingQueue timelineEntityQueue; @@ -300,6 +300,9 @@ public class TimelineV2ClientImpl extends TimelineV2Client { numberOfAsyncsToMerge = conf.getInt(YarnConfiguration.NUMBER_OF_ASYNC_ENTITIES_TO_MERGE, YarnConfiguration.DEFAULT_NUMBER_OF_ASYNC_ENTITIES_TO_MERGE); + drainTimeoutPeriod = conf.getLong( + YarnConfiguration.TIMELINE_V2_CLIENT_DRAIN_TIME_MILLIS, + YarnConfiguration.DEFAULT_TIMELINE_V2_CLIENT_DRAIN_TIME_MILLIS); } Runnable createRunnable() { @@ -330,7 +333,7 @@ public class TimelineV2ClientImpl extends TimelineV2Client { // Try to drain the remaining entities to be published @ the max for // 2 seconds long timeTillweDrain = - System.currentTimeMillis() + DRAIN_TIME_PERIOD; + System.currentTimeMillis() + drainTimeoutPeriod; while (!timelineEntityQueue.isEmpty()) { publishWithoutBlockingOnQueue(timelineEntityQueue.poll()); if (System.currentTimeMillis() > timeTillweDrain) { @@ -449,7 +452,7 @@ public class TimelineV2ClientImpl extends TimelineV2Client { LOG.info("Stopping TimelineClient."); executor.shutdownNow(); try { - executor.awaitTermination(DRAIN_TIME_PERIOD, TimeUnit.MILLISECONDS); + executor.awaitTermination(drainTimeoutPeriod, TimeUnit.MILLISECONDS); } catch (InterruptedException e) { Thread.currentThread().interrupt(); e.printStackTrace(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 727e2c9fbdb..bdd4de52c0a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -2101,6 +2101,15 @@ 1000 + + + The time period for which timeline v2 client will wait for draining + leftover entities after stop. + + yarn.timeline-service.client.drain-entities.timeout.ms + 2000 + + Enable timeline server to recover state after starting. If true, then yarn.timeline-service.state-store-class must be specified. From b58777a9c9a5b6f2e4bcfd2b3bede33f25f80dec Mon Sep 17 00:00:00 2001 From: Varun Saxena Date: Fri, 31 Mar 2017 02:17:20 +0530 Subject: [PATCH 169/188] YARN-6376. Exceptions caused by synchronous putEntities requests can be swallowed (Haibo Chen via Varun Saxena) --- .../timelineservice/collector/TimelineCollector.java | 10 ++++++++-- .../collector/TimelineCollectorManager.java | 7 ++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java index 353066bd775..4c9e9f85d2a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java @@ -137,8 +137,14 @@ public abstract class TimelineCollector extends CompositeService { + callerUgi + ")"); } - TimelineWriteResponse response = writeTimelineEntities(entities); - flushBufferedTimelineEntities(); + TimelineWriteResponse response; + // synchronize on the writer object so that no other threads can + // flush the writer buffer concurrently and swallow any exception + // caused by the timeline enitites that are being put here. + synchronized (writer) { + response = writeTimelineEntities(entities); + flushBufferedTimelineEntities(); + } return response; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java index 19896e82466..8ef9b43d517 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java @@ -259,7 +259,12 @@ public class TimelineCollectorManager extends AbstractService { public void run() { try { - writer.flush(); + // synchronize on the writer object to avoid flushing timeline + // entities placed on the buffer by synchronous putEntities + // requests. + synchronized (writer) { + writer.flush(); + } } catch (Throwable th) { // we need to handle all exceptions or subsequent execution may be // suppressed From e5f0622a6f40706d360d45200c8f259c79046438 Mon Sep 17 00:00:00 2001 From: Mingliang Liu Date: Thu, 30 Mar 2017 15:44:06 -0700 Subject: [PATCH 170/188] HDFS-11592. Closing a file has a wasteful preconditions in NameNode. Contributed by Eric Badger --- .../org/apache/hadoop/hdfs/server/namenode/INodeFile.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java index 2b0e0adea2a..3da6aa7c816 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java @@ -328,9 +328,11 @@ public class INodeFile extends INodeWithAdditionalFields for (int i = 0; i < blocks.length; i++) { final String err = checkBlockComplete(blocks, i, numCommittedAllowed, minReplication); - Preconditions.checkState(err == null, - "Unexpected block state: %s, file=%s (%s), blocks=%s (i=%s)", - err, this, getClass().getSimpleName(), Arrays.asList(blocks), i); + if(err != null) { + throw new IllegalStateException(String.format("Unexpected block state: " + + "%s, file=%s (%s), blocks=%s (i=%s)", err, this, + getClass().getSimpleName(), Arrays.asList(blocks), i)); + } } } From 064c8b25eca9bc825dc07a54d9147d65c9290a03 Mon Sep 17 00:00:00 2001 From: Yongjun Zhang Date: Thu, 30 Mar 2017 17:01:15 -0700 Subject: [PATCH 171/188] HADOOP-11794. Enable distcp to copy blocks in parallel. Contributed by Yongjun Zhang, Wei-Chiu Chuang, Xiao Chen. --- .../org/apache/hadoop/hdfs/DFSTestUtil.java | 22 +- .../org/apache/hadoop/tools/CopyListing.java | 37 +- .../hadoop/tools/CopyListingFileStatus.java | 87 ++++- .../java/org/apache/hadoop/tools/DistCp.java | 52 +++ .../hadoop/tools/DistCpOptionSwitch.java | 10 + .../apache/hadoop/tools/DistCpOptions.java | 22 +- .../apache/hadoop/tools/OptionsParser.java | 36 +- .../hadoop/tools/SimpleCopyListing.java | 83 ++-- .../hadoop/tools/mapred/CopyCommitter.java | 174 ++++++++- .../hadoop/tools/mapred/CopyMapper.java | 40 +- .../mapred/RetriableFileCopyCommand.java | 26 +- .../tools/mapred/UniformSizeInputFormat.java | 5 +- .../apache/hadoop/tools/util/DistCpUtils.java | 111 +++++- .../src/site/markdown/DistCp.md.vm | 1 + .../apache/hadoop/tools/TestDistCpSystem.java | 368 ++++++++++++++++-- .../hadoop/tools/TestOptionsParser.java | 2 +- .../tools/mapred/TestCopyCommitter.java | 5 +- 17 files changed, 971 insertions(+), 110 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java index 13291952a9e..9b782f32f2a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java @@ -862,7 +862,27 @@ public class DFSTestUtil { out.write(toAppend); } } - + + /** + * Append specified length of bytes to a given file, starting with new block. + * @param fs The file system + * @param p Path of the file to append + * @param length Length of bytes to append to the file + * @throws IOException + */ + public static void appendFileNewBlock(DistributedFileSystem fs, + Path p, int length) throws IOException { + assert fs.exists(p); + assert length >= 0; + byte[] toAppend = new byte[length]; + Random random = new Random(); + random.nextBytes(toAppend); + try (FSDataOutputStream out = fs.append(p, + EnumSet.of(CreateFlag.APPEND, CreateFlag.NEW_BLOCK), 4096, null)) { + out.write(toAppend); + } + } + /** * @return url content as string (UTF-8 encoding assumed) */ diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java index 481aa61b0f1..9ebf9d29b3b 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java @@ -145,12 +145,22 @@ public abstract class CopyListing extends Configured { Configuration config = getConf(); FileSystem fs = pathToListFile.getFileSystem(config); - Path sortedList = DistCpUtils.sortListing(fs, config, pathToListFile); + final boolean splitLargeFile = options.splitLargeFile(); + + // When splitLargeFile is enabled, we don't randomize the copylist + // earlier, so we don't do the sorting here. For a file that has + // multiple entries due to split, we check here that their + // is continuous. + // + Path checkPath = splitLargeFile? + pathToListFile : DistCpUtils.sortListing(fs, config, pathToListFile); SequenceFile.Reader reader = new SequenceFile.Reader( - config, SequenceFile.Reader.file(sortedList)); + config, SequenceFile.Reader.file(checkPath)); try { Text lastKey = new Text("*"); //source relative path can never hold * + long lastChunkOffset = -1; + long lastChunkLength = -1; CopyListingFileStatus lastFileStatus = new CopyListingFileStatus(); Text currentKey = new Text(); @@ -161,8 +171,21 @@ public abstract class CopyListing extends Configured { if (currentKey.equals(lastKey)) { CopyListingFileStatus currentFileStatus = new CopyListingFileStatus(); reader.getCurrentValue(currentFileStatus); - throw new DuplicateFileException("File " + lastFileStatus.getPath() + " and " + - currentFileStatus.getPath() + " would cause duplicates. Aborting"); + if (!splitLargeFile) { + throw new DuplicateFileException("File " + lastFileStatus.getPath() + + " and " + currentFileStatus.getPath() + + " would cause duplicates. Aborting"); + } else { + if (lastChunkOffset + lastChunkLength != + currentFileStatus.getChunkOffset()) { + throw new InvalidInputException("File " + lastFileStatus.getPath() + + " " + lastChunkOffset + "," + lastChunkLength + + " and " + currentFileStatus.getPath() + + " " + currentFileStatus.getChunkOffset() + "," + + currentFileStatus.getChunkLength() + + " are not continuous. Aborting"); + } + } } reader.getCurrentValue(lastFileStatus); if (options.shouldPreserve(DistCpOptions.FileAttribute.ACL)) { @@ -181,8 +204,12 @@ public abstract class CopyListing extends Configured { xAttrSupportCheckFsSet.add(lastFsUri); } } - lastKey.set(currentKey); + lastKey.set(currentKey); + if (splitLargeFile) { + lastChunkOffset = lastFileStatus.getChunkOffset(); + lastChunkLength = lastFileStatus.getChunkLength(); + } if (options.shouldUseDiff() && LOG.isDebugEnabled()) { LOG.debug("Copy list entry " + idx + ": " + lastFileStatus.getPath().toUri().getPath()); diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java index 00d4b325053..29c59ac1033 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java @@ -74,6 +74,14 @@ public final class CopyListingFileStatus implements Writable { private List aclEntries; private Map xAttrs; + // represents the offset and length of a file + // chunk in number of bytes. + // used when splitting a large file to chunks to copy in parallel. + // If a file is not large enough to split, chunkOffset would be 0 and + // chunkLength would be the length of the file. + private long chunkOffset = 0; + private long chunkLength = Long.MAX_VALUE; + /** * Default constructor. */ @@ -96,11 +104,32 @@ public final class CopyListingFileStatus implements Writable { fileStatus.getPath()); } + public CopyListingFileStatus(FileStatus fileStatus, + long chunkOffset, long chunkLength) { + this(fileStatus.getLen(), fileStatus.isDirectory(), + fileStatus.getReplication(), fileStatus.getBlockSize(), + fileStatus.getModificationTime(), fileStatus.getAccessTime(), + fileStatus.getPermission(), fileStatus.getOwner(), + fileStatus.getGroup(), + fileStatus.getPath()); + this.chunkOffset = chunkOffset; + this.chunkLength = chunkLength; + } + @SuppressWarnings("checkstyle:parameternumber") public CopyListingFileStatus(long length, boolean isdir, int blockReplication, long blocksize, long modificationTime, long accessTime, FsPermission permission, String owner, String group, Path path) { + this(length, isdir, blockReplication, blocksize, modificationTime, + accessTime, permission, owner, group, path, 0, Long.MAX_VALUE); + } + + @SuppressWarnings("checkstyle:parameternumber") + public CopyListingFileStatus(long length, boolean isdir, + int blockReplication, long blocksize, long modificationTime, + long accessTime, FsPermission permission, String owner, String group, + Path path, long chunkOffset, long chunkLength) { this.length = length; this.isdir = isdir; this.blockReplication = (short)blockReplication; @@ -117,6 +146,23 @@ public final class CopyListingFileStatus implements Writable { this.owner = (owner == null) ? "" : owner; this.group = (group == null) ? "" : group; this.path = path; + this.chunkOffset = chunkOffset; + this.chunkLength = chunkLength; + } + + public CopyListingFileStatus(CopyListingFileStatus other) { + this.length = other.length; + this.isdir = other.isdir; + this.blockReplication = other.blockReplication; + this.blocksize = other.blocksize; + this.modificationTime = other.modificationTime; + this.accessTime = other.accessTime; + this.permission = other.permission; + this.owner = other.owner; + this.group = other.group; + this.path = new Path(other.path.toUri()); + this.chunkOffset = other.chunkOffset; + this.chunkLength = other.chunkLength; } public Path getPath() { @@ -200,6 +246,31 @@ public final class CopyListingFileStatus implements Writable { this.xAttrs = xAttrs; } + public long getChunkOffset() { + return chunkOffset; + } + + public void setChunkOffset(long offset) { + this.chunkOffset = offset; + } + + public long getChunkLength() { + return chunkLength; + } + + public void setChunkLength(long chunkLength) { + this.chunkLength = chunkLength; + } + + public boolean isSplit() { + return getChunkLength() != Long.MAX_VALUE && + getChunkLength() != getLen(); + } + + public long getSizeToCopy() { + return isSplit()? getChunkLength() : getLen(); + } + @Override public void write(DataOutput out) throws IOException { Text.writeString(out, getPath().toString(), Text.DEFAULT_MAX_LEN); @@ -244,6 +315,9 @@ public final class CopyListingFileStatus implements Writable { } else { out.writeInt(NO_XATTRS); } + + out.writeLong(chunkOffset); + out.writeLong(chunkLength); } @Override @@ -292,6 +366,9 @@ public final class CopyListingFileStatus implements Writable { } else { xAttrs = null; } + + chunkOffset = in.readLong(); + chunkLength = in.readLong(); } @Override @@ -317,8 +394,14 @@ public final class CopyListingFileStatus implements Writable { public String toString() { StringBuilder sb = new StringBuilder(super.toString()); sb.append('{'); - sb.append("aclEntries = " + aclEntries); - sb.append(", xAttrs = " + xAttrs); + sb.append(this.getPath().toString()); + sb.append(" length = ").append(this.getLen()); + sb.append(" aclEntries = ").append(aclEntries); + sb.append(", xAttrs = ").append(xAttrs); + if (isSplit()) { + sb.append(", chunkOffset = ").append(this.getChunkOffset()); + sb.append(", chunkLength = ").append(this.getChunkLength()); + } sb.append('}'); return sb.toString(); } diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java index ab58e9c66d8..8c2fa24a15c 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java @@ -35,6 +35,7 @@ import org.apache.hadoop.mapreduce.Cluster; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.JobSubmissionFiles; +import org.apache.hadoop.tools.DistCpOptions.FileAttribute; import org.apache.hadoop.tools.CopyListing.*; import org.apache.hadoop.tools.mapred.CopyMapper; import org.apache.hadoop.tools.mapred.CopyOutputFormat; @@ -134,6 +135,7 @@ public class DistCp extends Configured implements Tool { try { inputOptions = (OptionsParser.parse(argv)); + setOptionsForSplitLargeFile(); setTargetPathExists(); LOG.info("Input Options: " + inputOptions); } catch (Throwable e) { @@ -235,6 +237,56 @@ public class DistCp extends Configured implements Tool { getConf().setBoolean(DistCpConstants.CONF_LABEL_TARGET_PATH_EXISTS, targetExists); } + + /** + * Check if concat is supported by fs. + * Throws UnsupportedOperationException if not. + */ + private void checkConcatSupport(FileSystem fs) { + try { + Path[] src = null; + Path tgt = null; + fs.concat(tgt, src); + } catch (UnsupportedOperationException use) { + throw new UnsupportedOperationException( + DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch() + + " is not supported since the target file system doesn't" + + " support concat.", use); + } catch (Exception e) { + // Ignore other exception + } + } + + /** + * Set up needed options for splitting large files. + */ + private void setOptionsForSplitLargeFile() throws IOException { + if (!inputOptions.splitLargeFile()) { + return; + } + Path target = inputOptions.getTargetPath(); + FileSystem targetFS = target.getFileSystem(getConf()); + checkConcatSupport(targetFS); + + LOG.info("Enabling preserving blocksize since " + + DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch() + " is passed."); + inputOptions.preserve(FileAttribute.BLOCKSIZE); + + LOG.info("Set " + + DistCpOptionSwitch.APPEND.getSwitch() + + " to false since " + DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch() + + " is passed."); + inputOptions.setAppend(false); + + LOG.info("Set " + + DistCpConstants.CONF_LABEL_SIMPLE_LISTING_RANDOMIZE_FILES + + " to false since " + DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch() + + " is passed."); + getConf().setBoolean( + DistCpConstants.CONF_LABEL_SIMPLE_LISTING_RANDOMIZE_FILES, false); + } + + /** * Create Job object for submitting it, with all the configuration * diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java index fb47d769233..ced9b540c89 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java @@ -169,6 +169,16 @@ public enum DistCpOptionSwitch { new Option("sizelimit", true, "(Deprecated!) Limit number of files " + "copied to <= n bytes")), + BLOCKS_PER_CHUNK("", + new Option("blocksperchunk", true, "If set to a positive value, files" + + "with more blocks than this value will be split into chunks of " + + " blocks to be transferred in parallel, and " + + "reassembled on the destination. By default, is " + + "0 and the files will be transmitted in their entirety without " + + "splitting. This switch is only applicable when the source file " + + "system implements getBlockLocations method and the target file " + + "system implements concat method")), + /** * Specify bandwidth per map in MB, accepts bandwidth as a fraction */ diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java index 8c37ff30ae0..9822d83dbd0 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java @@ -97,7 +97,11 @@ public class DistCpOptions { // targetPathExist is a derived field, it's initialized in the // beginning of distcp. private boolean targetPathExists = true; - + + // Size of chunk in number of blocks when splitting large file into chunks + // to copy in parallel. Default is 0 and file are not splitted. + private int blocksPerChunk = 0; + public static enum FileAttribute{ REPLICATION, BLOCKSIZE, USER, GROUP, PERMISSION, CHECKSUMTYPE, ACL, XATTR, TIMES; @@ -166,6 +170,7 @@ public class DistCpOptions { this.targetPath = that.getTargetPath(); this.targetPathExists = that.getTargetPathExists(); this.filtersFile = that.getFiltersFile(); + this.blocksPerChunk = that.blocksPerChunk; } } @@ -578,6 +583,18 @@ public class DistCpOptions { this.filtersFile = filtersFilename; } + public final void setBlocksPerChunk(int csize) { + this.blocksPerChunk = csize; + } + + public final int getBlocksPerChunk() { + return blocksPerChunk; + } + + public final boolean splitLargeFile() { + return blocksPerChunk > 0; + } + void validate() { if ((useDiff || useRdiff) && deleteMissing) { // -delete and -diff/-rdiff are mutually exclusive. For backward @@ -669,6 +686,8 @@ public class DistCpOptions { DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.FILTERS, filtersFile); } + DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.BLOCKS_PER_CHUNK, + String.valueOf(blocksPerChunk)); } /** @@ -704,6 +723,7 @@ public class DistCpOptions { ", targetPath=" + targetPath + ", targetPathExists=" + targetPathExists + ", filtersFile='" + filtersFile + '\'' + + ", blocksPerChunk=" + blocksPerChunk + '}'; } diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java index d0f82ca76b5..8881264d281 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java @@ -172,11 +172,44 @@ public class OptionsParser { DistCpOptionSwitch.FILTERS.getSwitch())); } + parseBlocksPerChunk(command, option); + option.validate(); return option; } + + /** + * A helper method to parse chunk size in number of blocks. + * Used when breaking large file into chunks to copy in parallel. + * + * @param command command line arguments + */ + private static void parseBlocksPerChunk(CommandLine command, + DistCpOptions option) { + boolean hasOption = + command.hasOption(DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch()); + LOG.info("parseChunkSize: " + + DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch() + " " + hasOption); + if (hasOption) { + String chunkSizeString = getVal(command, + DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch().trim()); + try { + int csize = Integer.parseInt(chunkSizeString); + if (csize < 0) { + csize = 0; + } + LOG.info("Set distcp blocksPerChunk to " + csize); + option.setBlocksPerChunk(csize); + } + catch (NumberFormatException e) { + throw new IllegalArgumentException("blocksPerChunk is invalid: " + + chunkSizeString, e); + } + } + } + /** * parseSizeLimit is a helper method for parsing the deprecated * argument SIZE_LIMIT. @@ -211,8 +244,7 @@ public class OptionsParser { DistCpOptionSwitch.FILE_LIMIT.getSwitch().trim()); try { Integer.parseInt(fileLimitString); - } - catch (NumberFormatException e) { + } catch (NumberFormatException e) { throw new IllegalArgumentException("File-limit is invalid: " + fileLimitString, e); } diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java index 105e4f2fe17..af913474550 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java @@ -19,6 +19,7 @@ package org.apache.hadoop.tools; import com.google.common.collect.Lists; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; @@ -47,6 +48,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Random; +import java.util.LinkedList; import static org.apache.hadoop.tools.DistCpConstants .HDFS_RESERVED_RAW_DIRECTORY_NAME; @@ -240,10 +242,10 @@ public class SimpleCopyListing extends CopyListing { final boolean preserveAcls = options.shouldPreserve(FileAttribute.ACL); final boolean preserveXAttrs = options.shouldPreserve(FileAttribute.XATTR); final boolean preserveRawXAttrs = options.shouldPreserveRawXattrs(); - CopyListingFileStatus fileCopyListingStatus = + LinkedList fileCopyListingStatus = DistCpUtils.toCopyListingFileStatus(sourceFS, fileStatus, - preserveAcls, preserveXAttrs, preserveRawXAttrs); - + preserveAcls, preserveXAttrs, preserveRawXAttrs, + options.getBlocksPerChunk()); writeToFileListingRoot(fileListWriter, fileCopyListingStatus, sourceRoot, options); } @@ -348,9 +350,10 @@ public class SimpleCopyListing extends CopyListing { FileStatus[] sourceFiles = sourceFS.listStatus(path); boolean explore = (sourceFiles != null && sourceFiles.length > 0); if (!explore || rootStatus.isDirectory()) { - CopyListingFileStatus rootCopyListingStatus = - DistCpUtils.toCopyListingFileStatus(sourceFS, rootStatus, - preserveAcls, preserveXAttrs, preserveRawXAttrs); + LinkedList rootCopyListingStatus = + DistCpUtils.toCopyListingFileStatus(sourceFS, rootStatus, + preserveAcls, preserveXAttrs, preserveRawXAttrs, + options.getBlocksPerChunk()); writeToFileListingRoot(fileListWriter, rootCopyListingStatus, sourcePathRoot, options); } @@ -360,20 +363,20 @@ public class SimpleCopyListing extends CopyListing { if (LOG.isDebugEnabled()) { LOG.debug("Recording source-path: " + sourceStatus.getPath() + " for copy."); } - CopyListingFileStatus sourceCopyListingStatus = - DistCpUtils.toCopyListingFileStatus(sourceFS, sourceStatus, - preserveAcls && sourceStatus.isDirectory(), - preserveXAttrs && sourceStatus.isDirectory(), - preserveRawXAttrs && sourceStatus.isDirectory()); - if (randomizeFileListing) { - addToFileListing(statusList, - new FileStatusInfo(sourceCopyListingStatus, sourcePathRoot), - fileListWriter); - } else { - writeToFileListing(fileListWriter, sourceCopyListingStatus, - sourcePathRoot); + LinkedList sourceCopyListingStatus = + DistCpUtils.toCopyListingFileStatus(sourceFS, sourceStatus, + preserveAcls && sourceStatus.isDirectory(), + preserveXAttrs && sourceStatus.isDirectory(), + preserveRawXAttrs && sourceStatus.isDirectory(), + options.getBlocksPerChunk()); + for (CopyListingFileStatus fs : sourceCopyListingStatus) { + if (randomizeFileListing) { + addToFileListing(statusList, + new FileStatusInfo(fs, sourcePathRoot), fileListWriter); + } else { + writeToFileListing(fileListWriter, fs, sourcePathRoot); + } } - if (sourceStatus.isDirectory()) { if (LOG.isDebugEnabled()) { LOG.debug("Adding source dir for traverse: " + sourceStatus.getPath()); @@ -641,18 +644,20 @@ public class SimpleCopyListing extends CopyListing { LOG.debug("Recording source-path: " + child.getPath() + " for copy."); } if (workResult.getSuccess()) { - CopyListingFileStatus childCopyListingStatus = + LinkedList childCopyListingStatus = DistCpUtils.toCopyListingFileStatus(sourceFS, child, preserveAcls && child.isDirectory(), preserveXAttrs && child.isDirectory(), - preserveRawXattrs && child.isDirectory()); - if (randomizeFileListing) { - addToFileListing(fileStatuses, - new FileStatusInfo(childCopyListingStatus, sourcePathRoot), - fileListWriter); - } else { - writeToFileListing(fileListWriter, childCopyListingStatus, - sourcePathRoot); + preserveRawXattrs && child.isDirectory(), + options.getBlocksPerChunk()); + + for (CopyListingFileStatus fs : childCopyListingStatus) { + if (randomizeFileListing) { + addToFileListing(fileStatuses, + new FileStatusInfo(fs, sourcePathRoot), fileListWriter); + } else { + writeToFileListing(fileListWriter, fs, sourcePathRoot); + } } } if (retry < maxRetries) { @@ -675,19 +680,21 @@ public class SimpleCopyListing extends CopyListing { } private void writeToFileListingRoot(SequenceFile.Writer fileListWriter, - CopyListingFileStatus fileStatus, Path sourcePathRoot, + LinkedList fileStatus, Path sourcePathRoot, DistCpOptions options) throws IOException { boolean syncOrOverwrite = options.shouldSyncFolder() || options.shouldOverwrite(); - if (fileStatus.getPath().equals(sourcePathRoot) && - fileStatus.isDirectory() && syncOrOverwrite) { - // Skip the root-paths when syncOrOverwrite - if (LOG.isDebugEnabled()) { - LOG.debug("Skip " + fileStatus.getPath()); - } - return; + for (CopyListingFileStatus fs : fileStatus) { + if (fs.getPath().equals(sourcePathRoot) && + fs.isDirectory() && syncOrOverwrite) { + // Skip the root-paths when syncOrOverwrite + if (LOG.isDebugEnabled()) { + LOG.debug("Skip " + fs.getPath()); + } + return; + } + writeToFileListing(fileListWriter, fs, sourcePathRoot); } - writeToFileListing(fileListWriter, fileStatus, sourcePathRoot); } private void writeToFileListing(SequenceFile.Writer fileListWriter, @@ -707,7 +714,7 @@ public class SimpleCopyListing extends CopyListing { fileListWriter.sync(); if (!fileStatus.isDirectory()) { - totalBytesToCopy += fileStatus.getLen(); + totalBytesToCopy += fileStatus.getSizeToCopy(); } else { totalDirs++; } diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java index 75cefb488ae..6ddaab99c3d 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java @@ -27,6 +27,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.Text; +import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.JobStatus; import org.apache.hadoop.mapreduce.TaskAttemptContext; @@ -34,14 +35,17 @@ import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter; import org.apache.hadoop.tools.CopyListing; import org.apache.hadoop.tools.CopyListingFileStatus; import org.apache.hadoop.tools.DistCpConstants; +import org.apache.hadoop.tools.DistCpOptionSwitch; import org.apache.hadoop.tools.DistCpOptions; import org.apache.hadoop.tools.DistCpOptions.FileAttribute; import org.apache.hadoop.tools.GlobbedCopyListing; import org.apache.hadoop.tools.util.DistCpUtils; +import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; import java.util.EnumSet; +import java.util.LinkedList; import java.util.List; /** @@ -63,7 +67,8 @@ public class CopyCommitter extends FileOutputCommitter { private boolean syncFolder = false; private boolean overwrite = false; private boolean targetPathExists = true; - + private boolean ignoreFailures = false; + /** * Create a output committer * @@ -82,8 +87,13 @@ public class CopyCommitter extends FileOutputCommitter { Configuration conf = jobContext.getConfiguration(); syncFolder = conf.getBoolean(DistCpConstants.CONF_LABEL_SYNC_FOLDERS, false); overwrite = conf.getBoolean(DistCpConstants.CONF_LABEL_OVERWRITE, false); - targetPathExists = conf.getBoolean(DistCpConstants.CONF_LABEL_TARGET_PATH_EXISTS, true); - + targetPathExists = conf.getBoolean( + DistCpConstants.CONF_LABEL_TARGET_PATH_EXISTS, true); + ignoreFailures = conf.getBoolean( + DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false); + + concatFileChunks(conf); + super.commitJob(jobContext); cleanupTempFiles(jobContext); @@ -169,9 +179,112 @@ public class CopyCommitter extends FileOutputCommitter { } } + private boolean isFileNotFoundException(IOException e) { + if (e instanceof FileNotFoundException) { + return true; + } + + if (e instanceof RemoteException) { + return ((RemoteException)e).unwrapRemoteException() + instanceof FileNotFoundException; + } + + return false; + } + + /** + * Concat chunk files for the same file into one. + * Iterate through copy listing, identify chunk files for the same file, + * concat them into one. + */ + private void concatFileChunks(Configuration conf) throws IOException { + + LOG.info("concat file chunks ..."); + + String spath = conf.get(DistCpConstants.CONF_LABEL_LISTING_FILE_PATH); + if (spath == null || spath.isEmpty()) { + return; + } + Path sourceListing = new Path(spath); + SequenceFile.Reader sourceReader = new SequenceFile.Reader(conf, + SequenceFile.Reader.file(sourceListing)); + Path targetRoot = + new Path(conf.get(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH)); + + try { + CopyListingFileStatus srcFileStatus = new CopyListingFileStatus(); + Text srcRelPath = new Text(); + CopyListingFileStatus lastFileStatus = null; + LinkedList allChunkPaths = new LinkedList(); + + // Iterate over every source path that was copied. + while (sourceReader.next(srcRelPath, srcFileStatus)) { + if (srcFileStatus.isDirectory()) { + continue; + } + Path targetFile = new Path(targetRoot.toString() + "/" + srcRelPath); + Path targetFileChunkPath = + DistCpUtils.getSplitChunkPath(targetFile, srcFileStatus); + if (LOG.isDebugEnabled()) { + LOG.debug(" add " + targetFileChunkPath + " to concat."); + } + allChunkPaths.add(targetFileChunkPath); + if (srcFileStatus.getChunkOffset() + srcFileStatus.getChunkLength() + == srcFileStatus.getLen()) { + // This is the last chunk of the splits, consolidate allChunkPaths + try { + concatFileChunks(conf, targetFile, allChunkPaths); + } catch (IOException e) { + // If the concat failed because a chunk file doesn't exist, + // then we assume that the CopyMapper has skipped copying this + // file, and we ignore the exception here. + // If a chunk file should have been created but it was not, then + // the CopyMapper would have failed. + if (!isFileNotFoundException(e)) { + String emsg = "Failed to concat chunk files for " + targetFile; + if (!ignoreFailures) { + throw new IOException(emsg, e); + } else { + LOG.warn(emsg, e); + } + } + } + allChunkPaths.clear(); + lastFileStatus = null; + } else { + if (lastFileStatus == null) { + lastFileStatus = new CopyListingFileStatus(srcFileStatus); + } else { + // Two neighboring chunks have to be consecutive ones for the same + // file, for them to be merged + if (!srcFileStatus.getPath().equals(lastFileStatus.getPath()) || + srcFileStatus.getChunkOffset() != + (lastFileStatus.getChunkOffset() + + lastFileStatus.getChunkLength())) { + String emsg = "Inconsistent sequence file: current " + + "chunk file " + srcFileStatus + " doesnt match prior " + + "entry " + lastFileStatus; + if (!ignoreFailures) { + throw new IOException(emsg); + } else { + LOG.warn(emsg + ", skipping concat this set."); + } + } else { + lastFileStatus.setChunkOffset(srcFileStatus.getChunkOffset()); + lastFileStatus.setChunkLength(srcFileStatus.getChunkLength()); + } + } + } + } + } finally { + IOUtils.closeStream(sourceReader); + } + } + // This method changes the target-directories' file-attributes (owner, // user/group permissions, etc.) based on the corresponding source directories. - private void preserveFileAttributesForDirectories(Configuration conf) throws IOException { + private void preserveFileAttributesForDirectories(Configuration conf) + throws IOException { String attrSymbols = conf.get(DistCpConstants.CONF_LABEL_PRESERVE_STATUS); final boolean syncOrOverwrite = syncFolder || overwrite; @@ -325,4 +438,57 @@ public class CopyCommitter extends FileOutputCommitter { ", Unable to move to " + finalDir); } } + + /** + * Concat the passed chunk files into one and rename it the targetFile. + */ + private void concatFileChunks(Configuration conf, Path targetFile, + LinkedList allChunkPaths) throws IOException { + if (allChunkPaths.size() == 1) { + return; + } + if (LOG.isDebugEnabled()) { + LOG.debug("concat " + targetFile + " allChunkSize+ " + + allChunkPaths.size()); + } + FileSystem dstfs = targetFile.getFileSystem(conf); + + Path firstChunkFile = allChunkPaths.removeFirst(); + Path[] restChunkFiles = new Path[allChunkPaths.size()]; + allChunkPaths.toArray(restChunkFiles); + if (LOG.isDebugEnabled()) { + LOG.debug("concat: firstchunk: " + dstfs.getFileStatus(firstChunkFile)); + int i = 0; + for (Path f : restChunkFiles) { + LOG.debug("concat: other chunk: " + i + ": " + dstfs.getFileStatus(f)); + ++i; + } + } + dstfs.concat(firstChunkFile, restChunkFiles); + if (LOG.isDebugEnabled()) { + LOG.debug("concat: result: " + dstfs.getFileStatus(firstChunkFile)); + } + rename(dstfs, firstChunkFile, targetFile); + } + + /** + * Rename tmp to dst on destFileSys. + * @param destFileSys the file ssystem + * @param tmp the source path + * @param dst the destination path + * @throws IOException if renaming failed + */ + private static void rename(FileSystem destFileSys, Path tmp, Path dst) + throws IOException { + try { + if (destFileSys.exists(dst)) { + destFileSys.delete(dst, true); + } + destFileSys.rename(tmp, dst); + } catch (IOException ioe) { + throw new IOException("Fail to rename tmp file (=" + tmp + + ") to destination file (=" + dst + ")", ioe); + } + } + } diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java index e1873f17e41..d6b3ba817b0 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java @@ -156,10 +156,12 @@ public class CopyMapper extends Mapper sourceFS = sourcePath.getFileSystem(conf); final boolean preserveXAttrs = fileAttributes.contains(FileAttribute.XATTR); - sourceCurrStatus = DistCpUtils.toCopyListingFileStatus(sourceFS, - sourceFS.getFileStatus(sourcePath), - fileAttributes.contains(FileAttribute.ACL), - preserveXAttrs, preserveRawXattrs); + sourceCurrStatus = DistCpUtils.toCopyListingFileStatusHelper(sourceFS, + sourceFS.getFileStatus(sourcePath), + fileAttributes.contains(FileAttribute.ACL), + preserveXAttrs, preserveRawXattrs, + sourceFileStatus.getChunkOffset(), + sourceFileStatus.getChunkLength()); } catch (FileNotFoundException e) { throw new IOException(new RetriableFileCopyCommand.CopyReadException(e)); } @@ -173,7 +175,8 @@ public class CopyMapper extends Mapper LOG.debug("Path could not be found: " + target, ignore); } - if (targetStatus != null && (targetStatus.isDirectory() != sourceCurrStatus.isDirectory())) { + if (targetStatus != null && + (targetStatus.isDirectory() != sourceCurrStatus.isDirectory())) { throw new IOException("Can't replace " + target + ". Target is " + getFileType(targetStatus) + ", Source is " + getFileType(sourceCurrStatus)); } @@ -183,19 +186,28 @@ public class CopyMapper extends Mapper return; } - FileAction action = checkUpdate(sourceFS, sourceCurrStatus, target, targetStatus); + FileAction action = checkUpdate(sourceFS, sourceCurrStatus, target, + targetStatus); + + Path tmpTarget = target; if (action == FileAction.SKIP) { LOG.info("Skipping copy of " + sourceCurrStatus.getPath() + " to " + target); updateSkipCounters(context, sourceCurrStatus); context.write(null, new Text("SKIP: " + sourceCurrStatus.getPath())); + } else { - copyFileWithRetry(description, sourceCurrStatus, target, context, + if (sourceCurrStatus.isSplit()) { + tmpTarget = DistCpUtils.getSplitChunkPath(target, sourceCurrStatus); + } + if (LOG.isDebugEnabled()) { + LOG.debug("copying " + sourceCurrStatus + " " + tmpTarget); + } + copyFileWithRetry(description, sourceCurrStatus, tmpTarget, context, action, fileAttributes); } - - DistCpUtils.preserve(target.getFileSystem(conf), target, sourceCurrStatus, - fileAttributes, preserveRawXattrs); + DistCpUtils.preserve(target.getFileSystem(conf), tmpTarget, + sourceCurrStatus, fileAttributes, preserveRawXattrs); } catch (IOException exception) { handleFailures(exception, sourceFileStatus, target, context); } @@ -261,8 +273,12 @@ public class CopyMapper extends Mapper private void handleFailures(IOException exception, CopyListingFileStatus sourceFileStatus, Path target, Context context) throws IOException, InterruptedException { - LOG.error("Failure in copying " + sourceFileStatus.getPath() + " to " + - target, exception); + LOG.error("Failure in copying " + sourceFileStatus.getPath() + + (sourceFileStatus.isSplit()? "," + + " offset=" + sourceFileStatus.getChunkOffset() + + " chunkLength=" + sourceFileStatus.getChunkLength() + : "") + + " to " + target, exception); if (ignoreFailures && ExceptionUtils.indexOfType(exception, CopyReadException.class) != -1) { diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java index 06acd78a8a1..2c17fef1a38 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java @@ -118,17 +118,21 @@ public class RetriableFileCopyCommand extends RetriableCommand { .contains(FileAttribute.CHECKSUMTYPE) ? sourceFS .getFileChecksum(sourcePath) : null; - final long offset = action == FileAction.APPEND ? targetFS.getFileStatus( - target).getLen() : 0; + long offset = (action == FileAction.APPEND) ? + targetFS.getFileStatus(target).getLen() : source.getChunkOffset(); long bytesRead = copyToFile(targetPath, targetFS, source, offset, context, fileAttributes, sourceChecksum); - compareFileLengths(source, targetPath, configuration, bytesRead - + offset); + if (!source.isSplit()) { + compareFileLengths(source, targetPath, configuration, bytesRead + + offset); + } //At this point, src&dest lengths are same. if length==0, we skip checksum if ((bytesRead != 0) && (!skipCrc)) { - compareCheckSums(sourceFS, source.getPath(), sourceChecksum, - targetFS, targetPath); + if (!source.isSplit()) { + compareCheckSums(sourceFS, source.getPath(), sourceChecksum, + targetFS, targetPath); + } } // it's not append case, thus we first write to a temporary file, rename // it to the target path. @@ -249,16 +253,26 @@ public class RetriableFileCopyCommand extends RetriableCommand { ThrottledInputStream inStream = null; long totalBytesRead = 0; + long chunkLength = source2.getChunkLength(); + boolean finished = false; try { inStream = getInputStream(source, context.getConfiguration()); int bytesRead = readBytes(inStream, buf, sourceOffset); while (bytesRead >= 0) { + if (chunkLength > 0 && + (totalBytesRead + bytesRead) >= chunkLength) { + bytesRead = (int)(chunkLength - totalBytesRead); + finished = true; + } totalBytesRead += bytesRead; if (action == FileAction.APPEND) { sourceOffset += bytesRead; } outStream.write(buf, 0, bytesRead); updateContextStatus(totalBytesRead, context, source2); + if (finished) { + break; + } bytesRead = readBytes(inStream, buf, sourceOffset); } outStream.close(); diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/UniformSizeInputFormat.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/UniformSizeInputFormat.java index 3e86d0931bc..d1c18ea8d16 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/UniformSizeInputFormat.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/UniformSizeInputFormat.java @@ -99,7 +99,8 @@ public class UniformSizeInputFormat while (reader.next(srcRelPath, srcFileStatus)) { // If adding the current file would cause the bytes per map to exceed // limit. Add the current file to new split - if (currentSplitSize + srcFileStatus.getLen() > nBytesPerSplit && lastPosition != 0) { + if (currentSplitSize + srcFileStatus.getChunkLength() > nBytesPerSplit + && lastPosition != 0) { FileSplit split = new FileSplit(listingFilePath, lastSplitStart, lastPosition - lastSplitStart, null); if (LOG.isDebugEnabled()) { @@ -109,7 +110,7 @@ public class UniformSizeInputFormat lastSplitStart = lastPosition; currentSplitSize = 0; } - currentSplitSize += srcFileStatus.getLen(); + currentSplitSize += srcFileStatus.getChunkLength(); lastPosition = reader.getPosition(); } if (lastPosition > lastSplitStart) { diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java index 76bc4c56268..e315b848de4 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java @@ -19,9 +19,11 @@ package org.apache.hadoop.tools.util; import com.google.common.collect.Maps; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -30,6 +32,7 @@ import org.apache.hadoop.fs.XAttr; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclUtil; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.InputFormat; @@ -44,6 +47,7 @@ import org.apache.hadoop.util.StringUtils; import java.io.IOException; import java.text.DecimalFormat; import java.util.EnumSet; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -116,7 +120,7 @@ public class DistCpUtils { * @return Class implementing the strategy specified in options. */ public static Class getStrategy(Configuration conf, - DistCpOptions options) { + DistCpOptions options) { String confLabel = "distcp." + StringUtils.toLowerCase(options.getCopyStrategy()) + ".strategy" + ".impl"; @@ -297,6 +301,86 @@ public class DistCpUtils { return fileSystem.getXAttrs(path); } + /** + * Converts FileStatus to a list of CopyListingFileStatus. + * The resulted list contains either one CopyListingFileStatus per chunk of + * file-blocks (if file-size exceeds blockSize * blocksPerChunk, and there + * are more blocks in the file than blocksperChunk), or a single + * CopyListingFileStatus for the entire file (if file-size is too small to + * split). + * If preserving ACLs, populates the CopyListingFileStatus with the ACLs. + * If preserving XAttrs, populates the CopyListingFileStatus with the XAttrs. + * + * @param fileSystem FileSystem containing the file + * @param fileStatus FileStatus of file + * @param preserveAcls boolean true if preserving ACLs + * @param preserveXAttrs boolean true if preserving XAttrs + * @param preserveRawXAttrs boolean true if preserving raw.* XAttrs + * @param blocksPerChunk size of chunks when copying chunks in parallel + * @return list of CopyListingFileStatus + * @throws IOException if there is an I/O error + */ + public static LinkedList toCopyListingFileStatus( + FileSystem fileSystem, FileStatus fileStatus, boolean preserveAcls, + boolean preserveXAttrs, boolean preserveRawXAttrs, int blocksPerChunk) + throws IOException { + LinkedList copyListingFileStatus = + new LinkedList(); + + final CopyListingFileStatus clfs = toCopyListingFileStatusHelper( + fileSystem, fileStatus, preserveAcls, + preserveXAttrs, preserveRawXAttrs, + 0, fileStatus.getLen()); + final long blockSize = fileStatus.getBlockSize(); + if (LOG.isDebugEnabled()) { + LOG.debug("toCopyListing: " + fileStatus + " chunkSize: " + + blocksPerChunk + " isDFS: " + + (fileSystem instanceof DistributedFileSystem)); + } + if ((blocksPerChunk > 0) && + !fileStatus.isDirectory() && + (fileStatus.getLen() > blockSize * blocksPerChunk)) { + // split only when the file size is larger than the intended chunk size + final BlockLocation[] blockLocations; + blockLocations = fileSystem.getFileBlockLocations(fileStatus, 0, + fileStatus.getLen()); + + int numBlocks = blockLocations.length; + long curPos = 0; + if (numBlocks <= blocksPerChunk) { + if (LOG.isDebugEnabled()) { + LOG.debug(" add file " + clfs); + } + copyListingFileStatus.add(clfs); + } else { + int i = 0; + while (i < numBlocks) { + long curLength = 0; + for (int j = 0; j < blocksPerChunk && i < numBlocks; ++j, ++i) { + curLength += blockLocations[i].getLength(); + } + if (curLength > 0) { + CopyListingFileStatus clfs1 = new CopyListingFileStatus(clfs); + clfs1.setChunkOffset(curPos); + clfs1.setChunkLength(curLength); + if (LOG.isDebugEnabled()) { + LOG.debug(" add file chunk " + clfs1); + } + copyListingFileStatus.add(clfs1); + curPos += curLength; + } + } + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug(" add file/dir " + clfs); + } + copyListingFileStatus.add(clfs); + } + + return copyListingFileStatus; + } + /** * Converts a FileStatus to a CopyListingFileStatus. If preserving ACLs, * populates the CopyListingFileStatus with the ACLs. If preserving XAttrs, @@ -307,13 +391,17 @@ public class DistCpUtils { * @param preserveAcls boolean true if preserving ACLs * @param preserveXAttrs boolean true if preserving XAttrs * @param preserveRawXAttrs boolean true if preserving raw.* XAttrs + * @param chunkOffset chunk offset in bytes + * @param chunkLength chunk length in bytes + * @return CopyListingFileStatus * @throws IOException if there is an I/O error */ - public static CopyListingFileStatus toCopyListingFileStatus( + public static CopyListingFileStatus toCopyListingFileStatusHelper( FileSystem fileSystem, FileStatus fileStatus, boolean preserveAcls, - boolean preserveXAttrs, boolean preserveRawXAttrs) throws IOException { + boolean preserveXAttrs, boolean preserveRawXAttrs, + long chunkOffset, long chunkLength) throws IOException { CopyListingFileStatus copyListingFileStatus = - new CopyListingFileStatus(fileStatus); + new CopyListingFileStatus(fileStatus, chunkOffset, chunkLength); if (preserveAcls) { FsPermission perm = fileStatus.getPermission(); if (perm.getAclBit()) { @@ -470,4 +558,19 @@ public class DistCpUtils { return (sourceChecksum == null || targetChecksum == null || sourceChecksum.equals(targetChecksum)); } + + /* + * Return the Path for a given chunk. + * Used when splitting large file into chunks to copy in parallel. + * @param targetFile path to target file + * @param srcFileStatus source file status in copy listing + * @return path to the chunk specified by the parameters to store + * in target cluster temporarily + */ + public static Path getSplitChunkPath(Path targetFile, + CopyListingFileStatus srcFileStatus) { + return new Path(targetFile.toString() + + ".____distcpSplit____" + srcFileStatus.getChunkOffset() + + "." + srcFileStatus.getChunkLength()); + } } diff --git a/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm b/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm index 41a6e94aeee..a77deb2ffee 100644 --- a/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm +++ b/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm @@ -237,6 +237,7 @@ Flag | Description | Notes `-rdiff ` | Use snapshot diff report between given two snapshots to identify what has been changed on the target since the snapshot `` was created on the target, and apply the diff reversely to the target, and copy modified files from the source's ``, to make the target the same as ``. | This option is valid only with `-update` option and the following conditions should be satisfied.
    1. Both the source and the target FileSystem must be DistributedFileSystem. The source and the target can be two different clusters/paths, or they can be exactly the same cluster/path. In the latter case, modified files are copied from target's `` to target's current state).
    2. Two snapshots `` and `` have been created on the target FS, and `` is older than ``. No change has been made on target since `` was created on the target.
    3. The source has the same snapshot ``, which has the same content as the `` on the target. All the files/directories in the target's `` are the same with source's ``.
    | `-numListstatusThreads` | Number of threads to use for building file listing | At most 40 threads. `-skipcrccheck` | Whether to skip CRC checks between source and target paths. | +`-blocksperchunk ` | Number of blocks per chunk. When specified, split files into chunks to copy in parallel | If set to a positive value, files with more blocks than this value will be split into chunks of `` blocks to be transferred in parallel, and reassembled on the destination. By default, `` is 0 and the files will be transmitted in their entirety without splitting. This switch is only applicable when the source file system implements getBlockLocations method and the target file system implements concat method. | Architecture of DistCp ---------------------- diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java index e3018a07730..b2266b3344d 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java @@ -23,17 +23,27 @@ import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.OutputStream; +import java.io.PrintStream; import java.net.URI; import java.util.ArrayList; import java.util.List; +import java.util.Random; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FsShell; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.util.ToolRunner; import org.junit.AfterClass; import org.junit.Assert; @@ -47,11 +57,15 @@ import org.junit.rules.Timeout; */ public class TestDistCpSystem { + private static final Log LOG = + LogFactory.getLog(TestDistCpSystem.class); + @Rule public Timeout globalTimeout = new Timeout(30000); private static final String SRCDAT = "srcdat"; private static final String DSTDAT = "dstdat"; + private static final long BLOCK_SIZE = 1024; private static MiniDFSCluster cluster; private static Configuration conf; @@ -63,27 +77,76 @@ public class TestDistCpSystem { this.path = path; this.isDir = isDir; } - String getPath() { return path; } - boolean isDirectory() { return isDir; } + + String getPath() { + return path; + } + + boolean isDirectory() { + return isDir; + } + } + + @BeforeClass + public static void beforeClass() throws IOException { + conf = new Configuration(); + conf.setLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, BLOCK_SIZE); + conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); + cluster.waitActive(); + } + + @AfterClass + public static void afterClass() throws IOException { + if (cluster != null) { + cluster.shutdown(); + } + } + + static String execCmd(FsShell shell, String... args) throws Exception { + ByteArrayOutputStream baout = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(baout, true); + PrintStream old = System.out; + System.setOut(out); + shell.run(args); + out.close(); + System.setOut(old); + return baout.toString(); } - private void createFiles(FileSystem fs, String topdir, - FileEntry[] entries) throws IOException { + private void createFiles(DistributedFileSystem fs, String topdir, + FileEntry[] entries, long chunkSize) throws IOException { + long seed = System.currentTimeMillis(); + Random rand = new Random(seed); + short replicationFactor = 2; for (FileEntry entry : entries) { - Path newpath = new Path(topdir + "/" + entry.getPath()); + Path newPath = new Path(topdir + "/" + entry.getPath()); if (entry.isDirectory()) { - fs.mkdirs(newpath); + fs.mkdirs(newPath); } else { - OutputStream out = fs.create(newpath); - try { - out.write((topdir + "/" + entry).getBytes()); - out.write("\n".getBytes()); - } finally { - out.close(); + long fileSize = BLOCK_SIZE *100; + int bufSize = 128; + if (chunkSize == -1) { + DFSTestUtil.createFile(fs, newPath, bufSize, + fileSize, BLOCK_SIZE, replicationFactor, seed); + } else { + // Create a variable length block file, by creating + // one block of half block size at the chunk boundary + long seg1 = chunkSize * BLOCK_SIZE - BLOCK_SIZE / 2; + long seg2 = fileSize - seg1; + DFSTestUtil.createFile(fs, newPath, bufSize, + seg1, BLOCK_SIZE, replicationFactor, seed); + DFSTestUtil.appendFileNewBlock(fs, newPath, (int)seg2); } } + seed = System.currentTimeMillis() + rand.nextLong(); } } + + private void createFiles(DistributedFileSystem fs, String topdir, + FileEntry[] entries) throws IOException { + createFiles(fs, topdir, entries, -1); + } private static FileStatus[] getFileStatus(FileSystem fs, String topdir, FileEntry[] files) throws IOException { @@ -104,18 +167,19 @@ public class TestDistCpSystem { } private void testPreserveUserHelper(String testRoot, - FileEntry[] srcEntries, - FileEntry[] dstEntries, - boolean createSrcDir, - boolean createTgtDir, - boolean update) throws Exception { + FileEntry[] srcEntries, + FileEntry[] dstEntries, + boolean createSrcDir, + boolean createTgtDir, + boolean update) throws Exception { final String testSrcRel = SRCDAT; final String testSrc = testRoot + "/" + testSrcRel; final String testDstRel = DSTDAT; final String testDst = testRoot + "/" + testDstRel; String nnUri = FileSystem.getDefaultUri(conf).toString(); - FileSystem fs = FileSystem.get(URI.create(nnUri), conf); + DistributedFileSystem fs = (DistributedFileSystem) + FileSystem.get(URI.create(nnUri), conf); fs.mkdirs(new Path(testRoot)); if (createSrcDir) { fs.mkdirs(new Path(testSrc)); @@ -129,8 +193,8 @@ public class TestDistCpSystem { for(int i = 0; i < srcEntries.length; i++) { fs.setOwner(srcstats[i].getPath(), "u" + i, null); } - String[] args = update? new String[]{"-pu", "-update", nnUri+testSrc, - nnUri+testDst} : new String[]{"-pu", nnUri+testSrc, nnUri+testDst}; + String[] args = update? new String[]{"-pub", "-update", nnUri+testSrc, + nnUri+testDst} : new String[]{"-pub", nnUri+testSrc, nnUri+testDst}; ToolRunner.run(conf, new DistCp(), args); @@ -145,20 +209,263 @@ public class TestDistCpSystem { deldir(fs, testRoot); } - @BeforeClass - public static void beforeClass() throws IOException { - conf = new Configuration(); - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); - cluster.waitActive(); + private void compareFiles(FileSystem fs, FileStatus srcStat, + FileStatus dstStat) throws Exception { + LOG.info("Comparing " + srcStat + " and " + dstStat); + assertEquals(srcStat.isDirectory(), dstStat.isDirectory()); + assertEquals(srcStat.getReplication(), dstStat.getReplication()); + assertEquals("File POSIX permission should match", + srcStat.getPermission(), dstStat.getPermission()); + assertEquals("File user ownership should match", + srcStat.getOwner(), dstStat.getOwner()); + assertEquals("File group ownership should match", + srcStat.getGroup(), dstStat.getGroup()); + // TODO; check ACL attributes + + if (srcStat.isDirectory()) { + return; + } + + assertEquals("File length should match (" + srcStat.getPath() + ")", + srcStat.getLen(), dstStat.getLen()); + + FSDataInputStream srcIn = fs.open(srcStat.getPath()); + FSDataInputStream dstIn = fs.open(dstStat.getPath()); + try { + byte[] readSrc = new byte[(int) + HdfsClientConfigKeys.DFS_BLOCK_SIZE_DEFAULT]; + byte[] readDst = new byte[(int) + HdfsClientConfigKeys.DFS_BLOCK_SIZE_DEFAULT]; + + int srcBytesRead = 0, tgtBytesRead = 0; + int srcIdx = 0, tgtIdx = 0; + long totalComparedBytes = 0; + while (true) { + if (srcBytesRead == 0) { + srcBytesRead = srcIn.read(readSrc); + srcIdx = 0; + } + if (tgtBytesRead == 0) { + tgtBytesRead = dstIn.read(readDst); + tgtIdx = 0; + } + if (srcBytesRead == 0 || tgtBytesRead == 0) { + LOG.info("______ compared src and dst files for " + + totalComparedBytes + " bytes, content match."); + if (srcBytesRead != tgtBytesRead) { + Assert.fail("Read mismatching size, compared " + + totalComparedBytes + " bytes between src and dst file " + + srcStat + " and " + dstStat); + } + if (totalComparedBytes != srcStat.getLen()) { + Assert.fail("Only read/compared " + totalComparedBytes + + " bytes between src and dst file " + srcStat + + " and " + dstStat); + } else { + // success + break; + } + } + for (; srcIdx < srcBytesRead && tgtIdx < tgtBytesRead; + ++srcIdx, ++tgtIdx) { + if (readSrc[srcIdx] != readDst[tgtIdx]) { + Assert.fail("src and dst file does not match at " + + totalComparedBytes + " between " + + srcStat + " and " + dstStat); + } + ++totalComparedBytes; + } + LOG.info("______ compared src and dst files for " + + totalComparedBytes + " bytes, content match. FileLength: " + + srcStat.getLen()); + if (totalComparedBytes == srcStat.getLen()) { + LOG.info("______ Final:" + srcIdx + " " + + srcBytesRead + " " + tgtIdx + " " + tgtBytesRead); + break; + } + if (srcIdx == srcBytesRead) { + srcBytesRead = 0; + } + if (tgtIdx == tgtBytesRead) { + tgtBytesRead = 0; + } + } + } finally { + if (srcIn != null) { + srcIn.close(); + } + if (dstIn != null) { + dstIn.close(); + } + } } - @AfterClass - public static void afterClass() throws IOException { - if (cluster != null) { - cluster.shutdown(); + // WC: needed because the current distcp does not create target dirs + private void createDestDir(FileSystem fs, String testDst, + FileStatus[] srcStats, FileEntry[] srcFiles) throws IOException { + fs.mkdirs(new Path(testDst)); + + for (int i=0; i=0; --i) { + if (!srcFiles[i].isDirectory()) { + LOG.info("Modifying " + srcStats[i].getPath()); + DFSTestUtil.appendFileNewBlock(fs, srcStats[i].getPath(), + (int)BLOCK_SIZE * 3); + break; + } + } + // get file status after modifying file + srcStats = getFileStatus(fs, testRoot, srcFiles); + + args = new String[] {"-pugp", "-update", "-blocksperchunk", + String.valueOf(chunkSize), + nnUri + testSrc, nnUri + testDst + "/" + testSrcRel}; + + copyAndVerify(fs, srcFiles, srcStats, testDst, args); + + deldir(fs, testRoot); + } + + @Test + public void testRecursiveChunkCopy() throws Exception { + FileEntry[] srcFiles = { + new FileEntry(SRCDAT, true), + new FileEntry(SRCDAT + "/file0", false), + new FileEntry(SRCDAT + "/dir1", true), + new FileEntry(SRCDAT + "/dir2", true), + new FileEntry(SRCDAT + "/dir1/file1", false) + }; + chunkCopy(srcFiles); + } + + @Test + public void testChunkCopyOneFile() throws Exception { + FileEntry[] srcFiles = { + new FileEntry(SRCDAT, true), + new FileEntry(SRCDAT + "/file0", false) + }; + chunkCopy(srcFiles); + } + + @Test + public void testDistcpLargeFile() throws Exception { + FileEntry[] srcfiles = { + new FileEntry(SRCDAT, true), + new FileEntry(SRCDAT + "/file", false) + }; + + final String testRoot = "/testdir"; + final String testSrcRel = SRCDAT; + final String testSrc = testRoot + "/" + testSrcRel; + final String testDstRel = DSTDAT; + final String testDst = testRoot + "/" + testDstRel; + + String nnUri = FileSystem.getDefaultUri(conf).toString(); + DistributedFileSystem fs = + (DistributedFileSystem) FileSystem.get(URI.create(nnUri), conf); + fs.mkdirs(new Path(testRoot)); + fs.mkdirs(new Path(testSrc)); + fs.mkdirs(new Path(testDst)); + long chunkSize = 6; + createFiles(fs, testRoot, srcfiles, chunkSize); + + String srcFileName = testRoot + Path.SEPARATOR + srcfiles[1].getPath(); + Path srcfile = new Path(srcFileName); + + if(!cluster.getFileSystem().exists(srcfile)){ + throw new Exception("src not exist"); + } + + final long srcLen = fs.getFileStatus(srcfile).getLen(); + + FileStatus[] srcstats = getFileStatus(fs, testRoot, srcfiles); + for (int i = 0; i < srcfiles.length; i++) { + fs.setOwner(srcstats[i].getPath(), "u" + i, null); + } + String[] args = new String[] { + "-blocksperchunk", + String.valueOf(chunkSize), + nnUri + testSrc, + nnUri + testDst + }; + + LOG.info("_____ running distcp: " + args[0] + " " + args[1]); + ToolRunner.run(conf, new DistCp(), args); + + String realTgtPath = testDst; + FileStatus[] dststat = getFileStatus(fs, realTgtPath, srcfiles); + assertEquals("File length should match", srcLen, + dststat[dststat.length - 1].getLen()); + + this.compareFiles(fs, srcstats[srcstats.length-1], + dststat[dststat.length-1]); + deldir(fs, testRoot); + } + @Test public void testPreserveUseNonEmptyDir() throws Exception { String testRoot = "/testdir." + getMethodName(); @@ -180,7 +487,6 @@ public class TestDistCpSystem { testPreserveUserHelper(testRoot, srcfiles, dstfiles, false, false, false); } - @Test public void testPreserveUserEmptyDir() throws Exception { String testRoot = "/testdir." + getMethodName(); diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java index efe46272e38..f94ba97ec34 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java @@ -394,7 +394,7 @@ public class TestOptionsParser { + "copyStrategy='uniformsize', preserveStatus=[], " + "preserveRawXattrs=false, atomicWorkPath=null, logPath=null, " + "sourceFileListing=abc, sourcePaths=null, targetPath=xyz, " - + "targetPathExists=true, filtersFile='null'}"; + + "targetPathExists=true, filtersFile='null', blocksPerChunk=0}"; String optionString = option.toString(); Assert.assertEquals(val, optionString); Assert.assertNotSame(DistCpOptionSwitch.ATOMIC_COMMIT.toString(), diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java index 2e9a350b1ca..2452d6fee3d 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java @@ -81,6 +81,10 @@ public class TestCopyCommitter { @Before public void createMetaFolder() { config.set(DistCpConstants.CONF_LABEL_META_FOLDER, "/meta"); + // Unset listing file path since the config is shared by + // multiple tests, and some test doesn't set it, such as + // testNoCommitAction, but the distcp code will check it. + config.set(DistCpConstants.CONF_LABEL_LISTING_FILE_PATH, ""); Path meta = new Path("/meta"); try { cluster.getFileSystem().mkdirs(meta); @@ -326,7 +330,6 @@ public class TestCopyCommitter { committer.commitJob(jobContext); Assert.assertFalse(fs.exists(new Path(workPath))); Assert.assertTrue(fs.exists(new Path(finalPath))); - } catch (IOException e) { LOG.error("Exception encountered while testing for preserve status", e); Assert.fail("Atomic commit failure"); From 144f1cf76527e6c75aec77ef683a898580f3cc8d Mon Sep 17 00:00:00 2001 From: Yongjun Zhang Date: Thu, 30 Mar 2017 17:38:18 -0700 Subject: [PATCH 172/188] Revert "HADOOP-11794. Enable distcp to copy blocks in parallel. Contributed by Yongjun Zhang, Wei-Chiu Chuang, Xiao Chen." This reverts commit 064c8b25eca9bc825dc07a54d9147d65c9290a03. --- .../org/apache/hadoop/hdfs/DFSTestUtil.java | 22 +- .../org/apache/hadoop/tools/CopyListing.java | 37 +- .../hadoop/tools/CopyListingFileStatus.java | 87 +---- .../java/org/apache/hadoop/tools/DistCp.java | 52 --- .../hadoop/tools/DistCpOptionSwitch.java | 10 - .../apache/hadoop/tools/DistCpOptions.java | 22 +- .../apache/hadoop/tools/OptionsParser.java | 36 +- .../hadoop/tools/SimpleCopyListing.java | 83 ++-- .../hadoop/tools/mapred/CopyCommitter.java | 174 +-------- .../hadoop/tools/mapred/CopyMapper.java | 40 +- .../mapred/RetriableFileCopyCommand.java | 26 +- .../tools/mapred/UniformSizeInputFormat.java | 5 +- .../apache/hadoop/tools/util/DistCpUtils.java | 111 +----- .../src/site/markdown/DistCp.md.vm | 1 - .../apache/hadoop/tools/TestDistCpSystem.java | 368 ++---------------- .../hadoop/tools/TestOptionsParser.java | 2 +- .../tools/mapred/TestCopyCommitter.java | 5 +- 17 files changed, 110 insertions(+), 971 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java index 9b782f32f2a..13291952a9e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java @@ -862,27 +862,7 @@ public class DFSTestUtil { out.write(toAppend); } } - - /** - * Append specified length of bytes to a given file, starting with new block. - * @param fs The file system - * @param p Path of the file to append - * @param length Length of bytes to append to the file - * @throws IOException - */ - public static void appendFileNewBlock(DistributedFileSystem fs, - Path p, int length) throws IOException { - assert fs.exists(p); - assert length >= 0; - byte[] toAppend = new byte[length]; - Random random = new Random(); - random.nextBytes(toAppend); - try (FSDataOutputStream out = fs.append(p, - EnumSet.of(CreateFlag.APPEND, CreateFlag.NEW_BLOCK), 4096, null)) { - out.write(toAppend); - } - } - + /** * @return url content as string (UTF-8 encoding assumed) */ diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java index 9ebf9d29b3b..481aa61b0f1 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java @@ -145,22 +145,12 @@ public abstract class CopyListing extends Configured { Configuration config = getConf(); FileSystem fs = pathToListFile.getFileSystem(config); - final boolean splitLargeFile = options.splitLargeFile(); - - // When splitLargeFile is enabled, we don't randomize the copylist - // earlier, so we don't do the sorting here. For a file that has - // multiple entries due to split, we check here that their - // is continuous. - // - Path checkPath = splitLargeFile? - pathToListFile : DistCpUtils.sortListing(fs, config, pathToListFile); + Path sortedList = DistCpUtils.sortListing(fs, config, pathToListFile); SequenceFile.Reader reader = new SequenceFile.Reader( - config, SequenceFile.Reader.file(checkPath)); + config, SequenceFile.Reader.file(sortedList)); try { Text lastKey = new Text("*"); //source relative path can never hold * - long lastChunkOffset = -1; - long lastChunkLength = -1; CopyListingFileStatus lastFileStatus = new CopyListingFileStatus(); Text currentKey = new Text(); @@ -171,21 +161,8 @@ public abstract class CopyListing extends Configured { if (currentKey.equals(lastKey)) { CopyListingFileStatus currentFileStatus = new CopyListingFileStatus(); reader.getCurrentValue(currentFileStatus); - if (!splitLargeFile) { - throw new DuplicateFileException("File " + lastFileStatus.getPath() - + " and " + currentFileStatus.getPath() - + " would cause duplicates. Aborting"); - } else { - if (lastChunkOffset + lastChunkLength != - currentFileStatus.getChunkOffset()) { - throw new InvalidInputException("File " + lastFileStatus.getPath() - + " " + lastChunkOffset + "," + lastChunkLength - + " and " + currentFileStatus.getPath() - + " " + currentFileStatus.getChunkOffset() + "," - + currentFileStatus.getChunkLength() - + " are not continuous. Aborting"); - } - } + throw new DuplicateFileException("File " + lastFileStatus.getPath() + " and " + + currentFileStatus.getPath() + " would cause duplicates. Aborting"); } reader.getCurrentValue(lastFileStatus); if (options.shouldPreserve(DistCpOptions.FileAttribute.ACL)) { @@ -204,12 +181,8 @@ public abstract class CopyListing extends Configured { xAttrSupportCheckFsSet.add(lastFsUri); } } - lastKey.set(currentKey); - if (splitLargeFile) { - lastChunkOffset = lastFileStatus.getChunkOffset(); - lastChunkLength = lastFileStatus.getChunkLength(); - } + if (options.shouldUseDiff() && LOG.isDebugEnabled()) { LOG.debug("Copy list entry " + idx + ": " + lastFileStatus.getPath().toUri().getPath()); diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java index 29c59ac1033..00d4b325053 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java @@ -74,14 +74,6 @@ public final class CopyListingFileStatus implements Writable { private List aclEntries; private Map xAttrs; - // represents the offset and length of a file - // chunk in number of bytes. - // used when splitting a large file to chunks to copy in parallel. - // If a file is not large enough to split, chunkOffset would be 0 and - // chunkLength would be the length of the file. - private long chunkOffset = 0; - private long chunkLength = Long.MAX_VALUE; - /** * Default constructor. */ @@ -104,32 +96,11 @@ public final class CopyListingFileStatus implements Writable { fileStatus.getPath()); } - public CopyListingFileStatus(FileStatus fileStatus, - long chunkOffset, long chunkLength) { - this(fileStatus.getLen(), fileStatus.isDirectory(), - fileStatus.getReplication(), fileStatus.getBlockSize(), - fileStatus.getModificationTime(), fileStatus.getAccessTime(), - fileStatus.getPermission(), fileStatus.getOwner(), - fileStatus.getGroup(), - fileStatus.getPath()); - this.chunkOffset = chunkOffset; - this.chunkLength = chunkLength; - } - @SuppressWarnings("checkstyle:parameternumber") public CopyListingFileStatus(long length, boolean isdir, int blockReplication, long blocksize, long modificationTime, long accessTime, FsPermission permission, String owner, String group, Path path) { - this(length, isdir, blockReplication, blocksize, modificationTime, - accessTime, permission, owner, group, path, 0, Long.MAX_VALUE); - } - - @SuppressWarnings("checkstyle:parameternumber") - public CopyListingFileStatus(long length, boolean isdir, - int blockReplication, long blocksize, long modificationTime, - long accessTime, FsPermission permission, String owner, String group, - Path path, long chunkOffset, long chunkLength) { this.length = length; this.isdir = isdir; this.blockReplication = (short)blockReplication; @@ -146,23 +117,6 @@ public final class CopyListingFileStatus implements Writable { this.owner = (owner == null) ? "" : owner; this.group = (group == null) ? "" : group; this.path = path; - this.chunkOffset = chunkOffset; - this.chunkLength = chunkLength; - } - - public CopyListingFileStatus(CopyListingFileStatus other) { - this.length = other.length; - this.isdir = other.isdir; - this.blockReplication = other.blockReplication; - this.blocksize = other.blocksize; - this.modificationTime = other.modificationTime; - this.accessTime = other.accessTime; - this.permission = other.permission; - this.owner = other.owner; - this.group = other.group; - this.path = new Path(other.path.toUri()); - this.chunkOffset = other.chunkOffset; - this.chunkLength = other.chunkLength; } public Path getPath() { @@ -246,31 +200,6 @@ public final class CopyListingFileStatus implements Writable { this.xAttrs = xAttrs; } - public long getChunkOffset() { - return chunkOffset; - } - - public void setChunkOffset(long offset) { - this.chunkOffset = offset; - } - - public long getChunkLength() { - return chunkLength; - } - - public void setChunkLength(long chunkLength) { - this.chunkLength = chunkLength; - } - - public boolean isSplit() { - return getChunkLength() != Long.MAX_VALUE && - getChunkLength() != getLen(); - } - - public long getSizeToCopy() { - return isSplit()? getChunkLength() : getLen(); - } - @Override public void write(DataOutput out) throws IOException { Text.writeString(out, getPath().toString(), Text.DEFAULT_MAX_LEN); @@ -315,9 +244,6 @@ public final class CopyListingFileStatus implements Writable { } else { out.writeInt(NO_XATTRS); } - - out.writeLong(chunkOffset); - out.writeLong(chunkLength); } @Override @@ -366,9 +292,6 @@ public final class CopyListingFileStatus implements Writable { } else { xAttrs = null; } - - chunkOffset = in.readLong(); - chunkLength = in.readLong(); } @Override @@ -394,14 +317,8 @@ public final class CopyListingFileStatus implements Writable { public String toString() { StringBuilder sb = new StringBuilder(super.toString()); sb.append('{'); - sb.append(this.getPath().toString()); - sb.append(" length = ").append(this.getLen()); - sb.append(" aclEntries = ").append(aclEntries); - sb.append(", xAttrs = ").append(xAttrs); - if (isSplit()) { - sb.append(", chunkOffset = ").append(this.getChunkOffset()); - sb.append(", chunkLength = ").append(this.getChunkLength()); - } + sb.append("aclEntries = " + aclEntries); + sb.append(", xAttrs = " + xAttrs); sb.append('}'); return sb.toString(); } diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java index 8c2fa24a15c..ab58e9c66d8 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java @@ -35,7 +35,6 @@ import org.apache.hadoop.mapreduce.Cluster; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.JobSubmissionFiles; -import org.apache.hadoop.tools.DistCpOptions.FileAttribute; import org.apache.hadoop.tools.CopyListing.*; import org.apache.hadoop.tools.mapred.CopyMapper; import org.apache.hadoop.tools.mapred.CopyOutputFormat; @@ -135,7 +134,6 @@ public class DistCp extends Configured implements Tool { try { inputOptions = (OptionsParser.parse(argv)); - setOptionsForSplitLargeFile(); setTargetPathExists(); LOG.info("Input Options: " + inputOptions); } catch (Throwable e) { @@ -237,56 +235,6 @@ public class DistCp extends Configured implements Tool { getConf().setBoolean(DistCpConstants.CONF_LABEL_TARGET_PATH_EXISTS, targetExists); } - - /** - * Check if concat is supported by fs. - * Throws UnsupportedOperationException if not. - */ - private void checkConcatSupport(FileSystem fs) { - try { - Path[] src = null; - Path tgt = null; - fs.concat(tgt, src); - } catch (UnsupportedOperationException use) { - throw new UnsupportedOperationException( - DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch() + - " is not supported since the target file system doesn't" + - " support concat.", use); - } catch (Exception e) { - // Ignore other exception - } - } - - /** - * Set up needed options for splitting large files. - */ - private void setOptionsForSplitLargeFile() throws IOException { - if (!inputOptions.splitLargeFile()) { - return; - } - Path target = inputOptions.getTargetPath(); - FileSystem targetFS = target.getFileSystem(getConf()); - checkConcatSupport(targetFS); - - LOG.info("Enabling preserving blocksize since " - + DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch() + " is passed."); - inputOptions.preserve(FileAttribute.BLOCKSIZE); - - LOG.info("Set " + - DistCpOptionSwitch.APPEND.getSwitch() - + " to false since " + DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch() - + " is passed."); - inputOptions.setAppend(false); - - LOG.info("Set " + - DistCpConstants.CONF_LABEL_SIMPLE_LISTING_RANDOMIZE_FILES - + " to false since " + DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch() - + " is passed."); - getConf().setBoolean( - DistCpConstants.CONF_LABEL_SIMPLE_LISTING_RANDOMIZE_FILES, false); - } - - /** * Create Job object for submitting it, with all the configuration * diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java index ced9b540c89..fb47d769233 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java @@ -169,16 +169,6 @@ public enum DistCpOptionSwitch { new Option("sizelimit", true, "(Deprecated!) Limit number of files " + "copied to <= n bytes")), - BLOCKS_PER_CHUNK("", - new Option("blocksperchunk", true, "If set to a positive value, files" - + "with more blocks than this value will be split into chunks of " - + " blocks to be transferred in parallel, and " - + "reassembled on the destination. By default, is " - + "0 and the files will be transmitted in their entirety without " - + "splitting. This switch is only applicable when the source file " - + "system implements getBlockLocations method and the target file " - + "system implements concat method")), - /** * Specify bandwidth per map in MB, accepts bandwidth as a fraction */ diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java index 9822d83dbd0..8c37ff30ae0 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java @@ -97,11 +97,7 @@ public class DistCpOptions { // targetPathExist is a derived field, it's initialized in the // beginning of distcp. private boolean targetPathExists = true; - - // Size of chunk in number of blocks when splitting large file into chunks - // to copy in parallel. Default is 0 and file are not splitted. - private int blocksPerChunk = 0; - + public static enum FileAttribute{ REPLICATION, BLOCKSIZE, USER, GROUP, PERMISSION, CHECKSUMTYPE, ACL, XATTR, TIMES; @@ -170,7 +166,6 @@ public class DistCpOptions { this.targetPath = that.getTargetPath(); this.targetPathExists = that.getTargetPathExists(); this.filtersFile = that.getFiltersFile(); - this.blocksPerChunk = that.blocksPerChunk; } } @@ -583,18 +578,6 @@ public class DistCpOptions { this.filtersFile = filtersFilename; } - public final void setBlocksPerChunk(int csize) { - this.blocksPerChunk = csize; - } - - public final int getBlocksPerChunk() { - return blocksPerChunk; - } - - public final boolean splitLargeFile() { - return blocksPerChunk > 0; - } - void validate() { if ((useDiff || useRdiff) && deleteMissing) { // -delete and -diff/-rdiff are mutually exclusive. For backward @@ -686,8 +669,6 @@ public class DistCpOptions { DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.FILTERS, filtersFile); } - DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.BLOCKS_PER_CHUNK, - String.valueOf(blocksPerChunk)); } /** @@ -723,7 +704,6 @@ public class DistCpOptions { ", targetPath=" + targetPath + ", targetPathExists=" + targetPathExists + ", filtersFile='" + filtersFile + '\'' + - ", blocksPerChunk=" + blocksPerChunk + '}'; } diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java index 8881264d281..d0f82ca76b5 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java @@ -172,44 +172,11 @@ public class OptionsParser { DistCpOptionSwitch.FILTERS.getSwitch())); } - parseBlocksPerChunk(command, option); - option.validate(); return option; } - - /** - * A helper method to parse chunk size in number of blocks. - * Used when breaking large file into chunks to copy in parallel. - * - * @param command command line arguments - */ - private static void parseBlocksPerChunk(CommandLine command, - DistCpOptions option) { - boolean hasOption = - command.hasOption(DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch()); - LOG.info("parseChunkSize: " + - DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch() + " " + hasOption); - if (hasOption) { - String chunkSizeString = getVal(command, - DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch().trim()); - try { - int csize = Integer.parseInt(chunkSizeString); - if (csize < 0) { - csize = 0; - } - LOG.info("Set distcp blocksPerChunk to " + csize); - option.setBlocksPerChunk(csize); - } - catch (NumberFormatException e) { - throw new IllegalArgumentException("blocksPerChunk is invalid: " - + chunkSizeString, e); - } - } - } - /** * parseSizeLimit is a helper method for parsing the deprecated * argument SIZE_LIMIT. @@ -244,7 +211,8 @@ public class OptionsParser { DistCpOptionSwitch.FILE_LIMIT.getSwitch().trim()); try { Integer.parseInt(fileLimitString); - } catch (NumberFormatException e) { + } + catch (NumberFormatException e) { throw new IllegalArgumentException("File-limit is invalid: " + fileLimitString, e); } diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java index af913474550..105e4f2fe17 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java @@ -19,7 +19,6 @@ package org.apache.hadoop.tools; import com.google.common.collect.Lists; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; @@ -48,7 +47,6 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Random; -import java.util.LinkedList; import static org.apache.hadoop.tools.DistCpConstants .HDFS_RESERVED_RAW_DIRECTORY_NAME; @@ -242,10 +240,10 @@ public class SimpleCopyListing extends CopyListing { final boolean preserveAcls = options.shouldPreserve(FileAttribute.ACL); final boolean preserveXAttrs = options.shouldPreserve(FileAttribute.XATTR); final boolean preserveRawXAttrs = options.shouldPreserveRawXattrs(); - LinkedList fileCopyListingStatus = + CopyListingFileStatus fileCopyListingStatus = DistCpUtils.toCopyListingFileStatus(sourceFS, fileStatus, - preserveAcls, preserveXAttrs, preserveRawXAttrs, - options.getBlocksPerChunk()); + preserveAcls, preserveXAttrs, preserveRawXAttrs); + writeToFileListingRoot(fileListWriter, fileCopyListingStatus, sourceRoot, options); } @@ -350,10 +348,9 @@ public class SimpleCopyListing extends CopyListing { FileStatus[] sourceFiles = sourceFS.listStatus(path); boolean explore = (sourceFiles != null && sourceFiles.length > 0); if (!explore || rootStatus.isDirectory()) { - LinkedList rootCopyListingStatus = - DistCpUtils.toCopyListingFileStatus(sourceFS, rootStatus, - preserveAcls, preserveXAttrs, preserveRawXAttrs, - options.getBlocksPerChunk()); + CopyListingFileStatus rootCopyListingStatus = + DistCpUtils.toCopyListingFileStatus(sourceFS, rootStatus, + preserveAcls, preserveXAttrs, preserveRawXAttrs); writeToFileListingRoot(fileListWriter, rootCopyListingStatus, sourcePathRoot, options); } @@ -363,20 +360,20 @@ public class SimpleCopyListing extends CopyListing { if (LOG.isDebugEnabled()) { LOG.debug("Recording source-path: " + sourceStatus.getPath() + " for copy."); } - LinkedList sourceCopyListingStatus = - DistCpUtils.toCopyListingFileStatus(sourceFS, sourceStatus, - preserveAcls && sourceStatus.isDirectory(), - preserveXAttrs && sourceStatus.isDirectory(), - preserveRawXAttrs && sourceStatus.isDirectory(), - options.getBlocksPerChunk()); - for (CopyListingFileStatus fs : sourceCopyListingStatus) { - if (randomizeFileListing) { - addToFileListing(statusList, - new FileStatusInfo(fs, sourcePathRoot), fileListWriter); - } else { - writeToFileListing(fileListWriter, fs, sourcePathRoot); - } + CopyListingFileStatus sourceCopyListingStatus = + DistCpUtils.toCopyListingFileStatus(sourceFS, sourceStatus, + preserveAcls && sourceStatus.isDirectory(), + preserveXAttrs && sourceStatus.isDirectory(), + preserveRawXAttrs && sourceStatus.isDirectory()); + if (randomizeFileListing) { + addToFileListing(statusList, + new FileStatusInfo(sourceCopyListingStatus, sourcePathRoot), + fileListWriter); + } else { + writeToFileListing(fileListWriter, sourceCopyListingStatus, + sourcePathRoot); } + if (sourceStatus.isDirectory()) { if (LOG.isDebugEnabled()) { LOG.debug("Adding source dir for traverse: " + sourceStatus.getPath()); @@ -644,20 +641,18 @@ public class SimpleCopyListing extends CopyListing { LOG.debug("Recording source-path: " + child.getPath() + " for copy."); } if (workResult.getSuccess()) { - LinkedList childCopyListingStatus = + CopyListingFileStatus childCopyListingStatus = DistCpUtils.toCopyListingFileStatus(sourceFS, child, preserveAcls && child.isDirectory(), preserveXAttrs && child.isDirectory(), - preserveRawXattrs && child.isDirectory(), - options.getBlocksPerChunk()); - - for (CopyListingFileStatus fs : childCopyListingStatus) { - if (randomizeFileListing) { - addToFileListing(fileStatuses, - new FileStatusInfo(fs, sourcePathRoot), fileListWriter); - } else { - writeToFileListing(fileListWriter, fs, sourcePathRoot); - } + preserveRawXattrs && child.isDirectory()); + if (randomizeFileListing) { + addToFileListing(fileStatuses, + new FileStatusInfo(childCopyListingStatus, sourcePathRoot), + fileListWriter); + } else { + writeToFileListing(fileListWriter, childCopyListingStatus, + sourcePathRoot); } } if (retry < maxRetries) { @@ -680,21 +675,19 @@ public class SimpleCopyListing extends CopyListing { } private void writeToFileListingRoot(SequenceFile.Writer fileListWriter, - LinkedList fileStatus, Path sourcePathRoot, + CopyListingFileStatus fileStatus, Path sourcePathRoot, DistCpOptions options) throws IOException { boolean syncOrOverwrite = options.shouldSyncFolder() || options.shouldOverwrite(); - for (CopyListingFileStatus fs : fileStatus) { - if (fs.getPath().equals(sourcePathRoot) && - fs.isDirectory() && syncOrOverwrite) { - // Skip the root-paths when syncOrOverwrite - if (LOG.isDebugEnabled()) { - LOG.debug("Skip " + fs.getPath()); - } - return; - } - writeToFileListing(fileListWriter, fs, sourcePathRoot); + if (fileStatus.getPath().equals(sourcePathRoot) && + fileStatus.isDirectory() && syncOrOverwrite) { + // Skip the root-paths when syncOrOverwrite + if (LOG.isDebugEnabled()) { + LOG.debug("Skip " + fileStatus.getPath()); + } + return; } + writeToFileListing(fileListWriter, fileStatus, sourcePathRoot); } private void writeToFileListing(SequenceFile.Writer fileListWriter, @@ -714,7 +707,7 @@ public class SimpleCopyListing extends CopyListing { fileListWriter.sync(); if (!fileStatus.isDirectory()) { - totalBytesToCopy += fileStatus.getSizeToCopy(); + totalBytesToCopy += fileStatus.getLen(); } else { totalDirs++; } diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java index 6ddaab99c3d..75cefb488ae 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java @@ -27,7 +27,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.Text; -import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.JobStatus; import org.apache.hadoop.mapreduce.TaskAttemptContext; @@ -35,17 +34,14 @@ import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter; import org.apache.hadoop.tools.CopyListing; import org.apache.hadoop.tools.CopyListingFileStatus; import org.apache.hadoop.tools.DistCpConstants; -import org.apache.hadoop.tools.DistCpOptionSwitch; import org.apache.hadoop.tools.DistCpOptions; import org.apache.hadoop.tools.DistCpOptions.FileAttribute; import org.apache.hadoop.tools.GlobbedCopyListing; import org.apache.hadoop.tools.util.DistCpUtils; -import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; import java.util.EnumSet; -import java.util.LinkedList; import java.util.List; /** @@ -67,8 +63,7 @@ public class CopyCommitter extends FileOutputCommitter { private boolean syncFolder = false; private boolean overwrite = false; private boolean targetPathExists = true; - private boolean ignoreFailures = false; - + /** * Create a output committer * @@ -87,13 +82,8 @@ public class CopyCommitter extends FileOutputCommitter { Configuration conf = jobContext.getConfiguration(); syncFolder = conf.getBoolean(DistCpConstants.CONF_LABEL_SYNC_FOLDERS, false); overwrite = conf.getBoolean(DistCpConstants.CONF_LABEL_OVERWRITE, false); - targetPathExists = conf.getBoolean( - DistCpConstants.CONF_LABEL_TARGET_PATH_EXISTS, true); - ignoreFailures = conf.getBoolean( - DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false); - - concatFileChunks(conf); - + targetPathExists = conf.getBoolean(DistCpConstants.CONF_LABEL_TARGET_PATH_EXISTS, true); + super.commitJob(jobContext); cleanupTempFiles(jobContext); @@ -179,112 +169,9 @@ public class CopyCommitter extends FileOutputCommitter { } } - private boolean isFileNotFoundException(IOException e) { - if (e instanceof FileNotFoundException) { - return true; - } - - if (e instanceof RemoteException) { - return ((RemoteException)e).unwrapRemoteException() - instanceof FileNotFoundException; - } - - return false; - } - - /** - * Concat chunk files for the same file into one. - * Iterate through copy listing, identify chunk files for the same file, - * concat them into one. - */ - private void concatFileChunks(Configuration conf) throws IOException { - - LOG.info("concat file chunks ..."); - - String spath = conf.get(DistCpConstants.CONF_LABEL_LISTING_FILE_PATH); - if (spath == null || spath.isEmpty()) { - return; - } - Path sourceListing = new Path(spath); - SequenceFile.Reader sourceReader = new SequenceFile.Reader(conf, - SequenceFile.Reader.file(sourceListing)); - Path targetRoot = - new Path(conf.get(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH)); - - try { - CopyListingFileStatus srcFileStatus = new CopyListingFileStatus(); - Text srcRelPath = new Text(); - CopyListingFileStatus lastFileStatus = null; - LinkedList allChunkPaths = new LinkedList(); - - // Iterate over every source path that was copied. - while (sourceReader.next(srcRelPath, srcFileStatus)) { - if (srcFileStatus.isDirectory()) { - continue; - } - Path targetFile = new Path(targetRoot.toString() + "/" + srcRelPath); - Path targetFileChunkPath = - DistCpUtils.getSplitChunkPath(targetFile, srcFileStatus); - if (LOG.isDebugEnabled()) { - LOG.debug(" add " + targetFileChunkPath + " to concat."); - } - allChunkPaths.add(targetFileChunkPath); - if (srcFileStatus.getChunkOffset() + srcFileStatus.getChunkLength() - == srcFileStatus.getLen()) { - // This is the last chunk of the splits, consolidate allChunkPaths - try { - concatFileChunks(conf, targetFile, allChunkPaths); - } catch (IOException e) { - // If the concat failed because a chunk file doesn't exist, - // then we assume that the CopyMapper has skipped copying this - // file, and we ignore the exception here. - // If a chunk file should have been created but it was not, then - // the CopyMapper would have failed. - if (!isFileNotFoundException(e)) { - String emsg = "Failed to concat chunk files for " + targetFile; - if (!ignoreFailures) { - throw new IOException(emsg, e); - } else { - LOG.warn(emsg, e); - } - } - } - allChunkPaths.clear(); - lastFileStatus = null; - } else { - if (lastFileStatus == null) { - lastFileStatus = new CopyListingFileStatus(srcFileStatus); - } else { - // Two neighboring chunks have to be consecutive ones for the same - // file, for them to be merged - if (!srcFileStatus.getPath().equals(lastFileStatus.getPath()) || - srcFileStatus.getChunkOffset() != - (lastFileStatus.getChunkOffset() + - lastFileStatus.getChunkLength())) { - String emsg = "Inconsistent sequence file: current " + - "chunk file " + srcFileStatus + " doesnt match prior " + - "entry " + lastFileStatus; - if (!ignoreFailures) { - throw new IOException(emsg); - } else { - LOG.warn(emsg + ", skipping concat this set."); - } - } else { - lastFileStatus.setChunkOffset(srcFileStatus.getChunkOffset()); - lastFileStatus.setChunkLength(srcFileStatus.getChunkLength()); - } - } - } - } - } finally { - IOUtils.closeStream(sourceReader); - } - } - // This method changes the target-directories' file-attributes (owner, // user/group permissions, etc.) based on the corresponding source directories. - private void preserveFileAttributesForDirectories(Configuration conf) - throws IOException { + private void preserveFileAttributesForDirectories(Configuration conf) throws IOException { String attrSymbols = conf.get(DistCpConstants.CONF_LABEL_PRESERVE_STATUS); final boolean syncOrOverwrite = syncFolder || overwrite; @@ -438,57 +325,4 @@ public class CopyCommitter extends FileOutputCommitter { ", Unable to move to " + finalDir); } } - - /** - * Concat the passed chunk files into one and rename it the targetFile. - */ - private void concatFileChunks(Configuration conf, Path targetFile, - LinkedList allChunkPaths) throws IOException { - if (allChunkPaths.size() == 1) { - return; - } - if (LOG.isDebugEnabled()) { - LOG.debug("concat " + targetFile + " allChunkSize+ " - + allChunkPaths.size()); - } - FileSystem dstfs = targetFile.getFileSystem(conf); - - Path firstChunkFile = allChunkPaths.removeFirst(); - Path[] restChunkFiles = new Path[allChunkPaths.size()]; - allChunkPaths.toArray(restChunkFiles); - if (LOG.isDebugEnabled()) { - LOG.debug("concat: firstchunk: " + dstfs.getFileStatus(firstChunkFile)); - int i = 0; - for (Path f : restChunkFiles) { - LOG.debug("concat: other chunk: " + i + ": " + dstfs.getFileStatus(f)); - ++i; - } - } - dstfs.concat(firstChunkFile, restChunkFiles); - if (LOG.isDebugEnabled()) { - LOG.debug("concat: result: " + dstfs.getFileStatus(firstChunkFile)); - } - rename(dstfs, firstChunkFile, targetFile); - } - - /** - * Rename tmp to dst on destFileSys. - * @param destFileSys the file ssystem - * @param tmp the source path - * @param dst the destination path - * @throws IOException if renaming failed - */ - private static void rename(FileSystem destFileSys, Path tmp, Path dst) - throws IOException { - try { - if (destFileSys.exists(dst)) { - destFileSys.delete(dst, true); - } - destFileSys.rename(tmp, dst); - } catch (IOException ioe) { - throw new IOException("Fail to rename tmp file (=" + tmp - + ") to destination file (=" + dst + ")", ioe); - } - } - } diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java index d6b3ba817b0..e1873f17e41 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java @@ -156,12 +156,10 @@ public class CopyMapper extends Mapper sourceFS = sourcePath.getFileSystem(conf); final boolean preserveXAttrs = fileAttributes.contains(FileAttribute.XATTR); - sourceCurrStatus = DistCpUtils.toCopyListingFileStatusHelper(sourceFS, - sourceFS.getFileStatus(sourcePath), - fileAttributes.contains(FileAttribute.ACL), - preserveXAttrs, preserveRawXattrs, - sourceFileStatus.getChunkOffset(), - sourceFileStatus.getChunkLength()); + sourceCurrStatus = DistCpUtils.toCopyListingFileStatus(sourceFS, + sourceFS.getFileStatus(sourcePath), + fileAttributes.contains(FileAttribute.ACL), + preserveXAttrs, preserveRawXattrs); } catch (FileNotFoundException e) { throw new IOException(new RetriableFileCopyCommand.CopyReadException(e)); } @@ -175,8 +173,7 @@ public class CopyMapper extends Mapper LOG.debug("Path could not be found: " + target, ignore); } - if (targetStatus != null && - (targetStatus.isDirectory() != sourceCurrStatus.isDirectory())) { + if (targetStatus != null && (targetStatus.isDirectory() != sourceCurrStatus.isDirectory())) { throw new IOException("Can't replace " + target + ". Target is " + getFileType(targetStatus) + ", Source is " + getFileType(sourceCurrStatus)); } @@ -186,28 +183,19 @@ public class CopyMapper extends Mapper return; } - FileAction action = checkUpdate(sourceFS, sourceCurrStatus, target, - targetStatus); - - Path tmpTarget = target; + FileAction action = checkUpdate(sourceFS, sourceCurrStatus, target, targetStatus); if (action == FileAction.SKIP) { LOG.info("Skipping copy of " + sourceCurrStatus.getPath() + " to " + target); updateSkipCounters(context, sourceCurrStatus); context.write(null, new Text("SKIP: " + sourceCurrStatus.getPath())); - } else { - if (sourceCurrStatus.isSplit()) { - tmpTarget = DistCpUtils.getSplitChunkPath(target, sourceCurrStatus); - } - if (LOG.isDebugEnabled()) { - LOG.debug("copying " + sourceCurrStatus + " " + tmpTarget); - } - copyFileWithRetry(description, sourceCurrStatus, tmpTarget, context, + copyFileWithRetry(description, sourceCurrStatus, target, context, action, fileAttributes); } - DistCpUtils.preserve(target.getFileSystem(conf), tmpTarget, - sourceCurrStatus, fileAttributes, preserveRawXattrs); + + DistCpUtils.preserve(target.getFileSystem(conf), target, sourceCurrStatus, + fileAttributes, preserveRawXattrs); } catch (IOException exception) { handleFailures(exception, sourceFileStatus, target, context); } @@ -273,12 +261,8 @@ public class CopyMapper extends Mapper private void handleFailures(IOException exception, CopyListingFileStatus sourceFileStatus, Path target, Context context) throws IOException, InterruptedException { - LOG.error("Failure in copying " + sourceFileStatus.getPath() + - (sourceFileStatus.isSplit()? "," - + " offset=" + sourceFileStatus.getChunkOffset() - + " chunkLength=" + sourceFileStatus.getChunkLength() - : "") + - " to " + target, exception); + LOG.error("Failure in copying " + sourceFileStatus.getPath() + " to " + + target, exception); if (ignoreFailures && ExceptionUtils.indexOfType(exception, CopyReadException.class) != -1) { diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java index 2c17fef1a38..06acd78a8a1 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java @@ -118,21 +118,17 @@ public class RetriableFileCopyCommand extends RetriableCommand { .contains(FileAttribute.CHECKSUMTYPE) ? sourceFS .getFileChecksum(sourcePath) : null; - long offset = (action == FileAction.APPEND) ? - targetFS.getFileStatus(target).getLen() : source.getChunkOffset(); + final long offset = action == FileAction.APPEND ? targetFS.getFileStatus( + target).getLen() : 0; long bytesRead = copyToFile(targetPath, targetFS, source, offset, context, fileAttributes, sourceChecksum); - if (!source.isSplit()) { - compareFileLengths(source, targetPath, configuration, bytesRead - + offset); - } + compareFileLengths(source, targetPath, configuration, bytesRead + + offset); //At this point, src&dest lengths are same. if length==0, we skip checksum if ((bytesRead != 0) && (!skipCrc)) { - if (!source.isSplit()) { - compareCheckSums(sourceFS, source.getPath(), sourceChecksum, - targetFS, targetPath); - } + compareCheckSums(sourceFS, source.getPath(), sourceChecksum, + targetFS, targetPath); } // it's not append case, thus we first write to a temporary file, rename // it to the target path. @@ -253,26 +249,16 @@ public class RetriableFileCopyCommand extends RetriableCommand { ThrottledInputStream inStream = null; long totalBytesRead = 0; - long chunkLength = source2.getChunkLength(); - boolean finished = false; try { inStream = getInputStream(source, context.getConfiguration()); int bytesRead = readBytes(inStream, buf, sourceOffset); while (bytesRead >= 0) { - if (chunkLength > 0 && - (totalBytesRead + bytesRead) >= chunkLength) { - bytesRead = (int)(chunkLength - totalBytesRead); - finished = true; - } totalBytesRead += bytesRead; if (action == FileAction.APPEND) { sourceOffset += bytesRead; } outStream.write(buf, 0, bytesRead); updateContextStatus(totalBytesRead, context, source2); - if (finished) { - break; - } bytesRead = readBytes(inStream, buf, sourceOffset); } outStream.close(); diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/UniformSizeInputFormat.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/UniformSizeInputFormat.java index d1c18ea8d16..3e86d0931bc 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/UniformSizeInputFormat.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/UniformSizeInputFormat.java @@ -99,8 +99,7 @@ public class UniformSizeInputFormat while (reader.next(srcRelPath, srcFileStatus)) { // If adding the current file would cause the bytes per map to exceed // limit. Add the current file to new split - if (currentSplitSize + srcFileStatus.getChunkLength() > nBytesPerSplit - && lastPosition != 0) { + if (currentSplitSize + srcFileStatus.getLen() > nBytesPerSplit && lastPosition != 0) { FileSplit split = new FileSplit(listingFilePath, lastSplitStart, lastPosition - lastSplitStart, null); if (LOG.isDebugEnabled()) { @@ -110,7 +109,7 @@ public class UniformSizeInputFormat lastSplitStart = lastPosition; currentSplitSize = 0; } - currentSplitSize += srcFileStatus.getChunkLength(); + currentSplitSize += srcFileStatus.getLen(); lastPosition = reader.getPosition(); } if (lastPosition > lastSplitStart) { diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java index e315b848de4..76bc4c56268 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java @@ -19,11 +19,9 @@ package org.apache.hadoop.tools.util; import com.google.common.collect.Maps; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -32,7 +30,6 @@ import org.apache.hadoop.fs.XAttr; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclUtil; import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.InputFormat; @@ -47,7 +44,6 @@ import org.apache.hadoop.util.StringUtils; import java.io.IOException; import java.text.DecimalFormat; import java.util.EnumSet; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -120,7 +116,7 @@ public class DistCpUtils { * @return Class implementing the strategy specified in options. */ public static Class getStrategy(Configuration conf, - DistCpOptions options) { + DistCpOptions options) { String confLabel = "distcp." + StringUtils.toLowerCase(options.getCopyStrategy()) + ".strategy" + ".impl"; @@ -301,86 +297,6 @@ public class DistCpUtils { return fileSystem.getXAttrs(path); } - /** - * Converts FileStatus to a list of CopyListingFileStatus. - * The resulted list contains either one CopyListingFileStatus per chunk of - * file-blocks (if file-size exceeds blockSize * blocksPerChunk, and there - * are more blocks in the file than blocksperChunk), or a single - * CopyListingFileStatus for the entire file (if file-size is too small to - * split). - * If preserving ACLs, populates the CopyListingFileStatus with the ACLs. - * If preserving XAttrs, populates the CopyListingFileStatus with the XAttrs. - * - * @param fileSystem FileSystem containing the file - * @param fileStatus FileStatus of file - * @param preserveAcls boolean true if preserving ACLs - * @param preserveXAttrs boolean true if preserving XAttrs - * @param preserveRawXAttrs boolean true if preserving raw.* XAttrs - * @param blocksPerChunk size of chunks when copying chunks in parallel - * @return list of CopyListingFileStatus - * @throws IOException if there is an I/O error - */ - public static LinkedList toCopyListingFileStatus( - FileSystem fileSystem, FileStatus fileStatus, boolean preserveAcls, - boolean preserveXAttrs, boolean preserveRawXAttrs, int blocksPerChunk) - throws IOException { - LinkedList copyListingFileStatus = - new LinkedList(); - - final CopyListingFileStatus clfs = toCopyListingFileStatusHelper( - fileSystem, fileStatus, preserveAcls, - preserveXAttrs, preserveRawXAttrs, - 0, fileStatus.getLen()); - final long blockSize = fileStatus.getBlockSize(); - if (LOG.isDebugEnabled()) { - LOG.debug("toCopyListing: " + fileStatus + " chunkSize: " - + blocksPerChunk + " isDFS: " + - (fileSystem instanceof DistributedFileSystem)); - } - if ((blocksPerChunk > 0) && - !fileStatus.isDirectory() && - (fileStatus.getLen() > blockSize * blocksPerChunk)) { - // split only when the file size is larger than the intended chunk size - final BlockLocation[] blockLocations; - blockLocations = fileSystem.getFileBlockLocations(fileStatus, 0, - fileStatus.getLen()); - - int numBlocks = blockLocations.length; - long curPos = 0; - if (numBlocks <= blocksPerChunk) { - if (LOG.isDebugEnabled()) { - LOG.debug(" add file " + clfs); - } - copyListingFileStatus.add(clfs); - } else { - int i = 0; - while (i < numBlocks) { - long curLength = 0; - for (int j = 0; j < blocksPerChunk && i < numBlocks; ++j, ++i) { - curLength += blockLocations[i].getLength(); - } - if (curLength > 0) { - CopyListingFileStatus clfs1 = new CopyListingFileStatus(clfs); - clfs1.setChunkOffset(curPos); - clfs1.setChunkLength(curLength); - if (LOG.isDebugEnabled()) { - LOG.debug(" add file chunk " + clfs1); - } - copyListingFileStatus.add(clfs1); - curPos += curLength; - } - } - } - } else { - if (LOG.isDebugEnabled()) { - LOG.debug(" add file/dir " + clfs); - } - copyListingFileStatus.add(clfs); - } - - return copyListingFileStatus; - } - /** * Converts a FileStatus to a CopyListingFileStatus. If preserving ACLs, * populates the CopyListingFileStatus with the ACLs. If preserving XAttrs, @@ -391,17 +307,13 @@ public class DistCpUtils { * @param preserveAcls boolean true if preserving ACLs * @param preserveXAttrs boolean true if preserving XAttrs * @param preserveRawXAttrs boolean true if preserving raw.* XAttrs - * @param chunkOffset chunk offset in bytes - * @param chunkLength chunk length in bytes - * @return CopyListingFileStatus * @throws IOException if there is an I/O error */ - public static CopyListingFileStatus toCopyListingFileStatusHelper( + public static CopyListingFileStatus toCopyListingFileStatus( FileSystem fileSystem, FileStatus fileStatus, boolean preserveAcls, - boolean preserveXAttrs, boolean preserveRawXAttrs, - long chunkOffset, long chunkLength) throws IOException { + boolean preserveXAttrs, boolean preserveRawXAttrs) throws IOException { CopyListingFileStatus copyListingFileStatus = - new CopyListingFileStatus(fileStatus, chunkOffset, chunkLength); + new CopyListingFileStatus(fileStatus); if (preserveAcls) { FsPermission perm = fileStatus.getPermission(); if (perm.getAclBit()) { @@ -558,19 +470,4 @@ public class DistCpUtils { return (sourceChecksum == null || targetChecksum == null || sourceChecksum.equals(targetChecksum)); } - - /* - * Return the Path for a given chunk. - * Used when splitting large file into chunks to copy in parallel. - * @param targetFile path to target file - * @param srcFileStatus source file status in copy listing - * @return path to the chunk specified by the parameters to store - * in target cluster temporarily - */ - public static Path getSplitChunkPath(Path targetFile, - CopyListingFileStatus srcFileStatus) { - return new Path(targetFile.toString() - + ".____distcpSplit____" + srcFileStatus.getChunkOffset() - + "." + srcFileStatus.getChunkLength()); - } } diff --git a/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm b/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm index a77deb2ffee..41a6e94aeee 100644 --- a/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm +++ b/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm @@ -237,7 +237,6 @@ Flag | Description | Notes `-rdiff ` | Use snapshot diff report between given two snapshots to identify what has been changed on the target since the snapshot `` was created on the target, and apply the diff reversely to the target, and copy modified files from the source's ``, to make the target the same as ``. | This option is valid only with `-update` option and the following conditions should be satisfied.
    1. Both the source and the target FileSystem must be DistributedFileSystem. The source and the target can be two different clusters/paths, or they can be exactly the same cluster/path. In the latter case, modified files are copied from target's `` to target's current state).
    2. Two snapshots `` and `` have been created on the target FS, and `` is older than ``. No change has been made on target since `` was created on the target.
    3. The source has the same snapshot ``, which has the same content as the `` on the target. All the files/directories in the target's `` are the same with source's ``.
    | `-numListstatusThreads` | Number of threads to use for building file listing | At most 40 threads. `-skipcrccheck` | Whether to skip CRC checks between source and target paths. | -`-blocksperchunk ` | Number of blocks per chunk. When specified, split files into chunks to copy in parallel | If set to a positive value, files with more blocks than this value will be split into chunks of `` blocks to be transferred in parallel, and reassembled on the destination. By default, `` is 0 and the files will be transmitted in their entirety without splitting. This switch is only applicable when the source file system implements getBlockLocations method and the target file system implements concat method. | Architecture of DistCp ---------------------- diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java index b2266b3344d..e3018a07730 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java @@ -23,27 +23,17 @@ import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.PrintStream; +import java.io.OutputStream; import java.net.URI; import java.util.ArrayList; import java.util.List; -import java.util.Random; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FsShell; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.hdfs.DFSTestUtil; -import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.util.ToolRunner; import org.junit.AfterClass; import org.junit.Assert; @@ -57,15 +47,11 @@ import org.junit.rules.Timeout; */ public class TestDistCpSystem { - private static final Log LOG = - LogFactory.getLog(TestDistCpSystem.class); - @Rule public Timeout globalTimeout = new Timeout(30000); private static final String SRCDAT = "srcdat"; private static final String DSTDAT = "dstdat"; - private static final long BLOCK_SIZE = 1024; private static MiniDFSCluster cluster; private static Configuration conf; @@ -77,76 +63,27 @@ public class TestDistCpSystem { this.path = path; this.isDir = isDir; } - - String getPath() { - return path; - } - - boolean isDirectory() { - return isDir; - } - } - - @BeforeClass - public static void beforeClass() throws IOException { - conf = new Configuration(); - conf.setLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, BLOCK_SIZE); - conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE); - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); - cluster.waitActive(); - } - - @AfterClass - public static void afterClass() throws IOException { - if (cluster != null) { - cluster.shutdown(); - } - } - - static String execCmd(FsShell shell, String... args) throws Exception { - ByteArrayOutputStream baout = new ByteArrayOutputStream(); - PrintStream out = new PrintStream(baout, true); - PrintStream old = System.out; - System.setOut(out); - shell.run(args); - out.close(); - System.setOut(old); - return baout.toString(); + String getPath() { return path; } + boolean isDirectory() { return isDir; } } - private void createFiles(DistributedFileSystem fs, String topdir, - FileEntry[] entries, long chunkSize) throws IOException { - long seed = System.currentTimeMillis(); - Random rand = new Random(seed); - short replicationFactor = 2; + private void createFiles(FileSystem fs, String topdir, + FileEntry[] entries) throws IOException { for (FileEntry entry : entries) { - Path newPath = new Path(topdir + "/" + entry.getPath()); + Path newpath = new Path(topdir + "/" + entry.getPath()); if (entry.isDirectory()) { - fs.mkdirs(newPath); + fs.mkdirs(newpath); } else { - long fileSize = BLOCK_SIZE *100; - int bufSize = 128; - if (chunkSize == -1) { - DFSTestUtil.createFile(fs, newPath, bufSize, - fileSize, BLOCK_SIZE, replicationFactor, seed); - } else { - // Create a variable length block file, by creating - // one block of half block size at the chunk boundary - long seg1 = chunkSize * BLOCK_SIZE - BLOCK_SIZE / 2; - long seg2 = fileSize - seg1; - DFSTestUtil.createFile(fs, newPath, bufSize, - seg1, BLOCK_SIZE, replicationFactor, seed); - DFSTestUtil.appendFileNewBlock(fs, newPath, (int)seg2); + OutputStream out = fs.create(newpath); + try { + out.write((topdir + "/" + entry).getBytes()); + out.write("\n".getBytes()); + } finally { + out.close(); } } - seed = System.currentTimeMillis() + rand.nextLong(); } } - - private void createFiles(DistributedFileSystem fs, String topdir, - FileEntry[] entries) throws IOException { - createFiles(fs, topdir, entries, -1); - } private static FileStatus[] getFileStatus(FileSystem fs, String topdir, FileEntry[] files) throws IOException { @@ -167,19 +104,18 @@ public class TestDistCpSystem { } private void testPreserveUserHelper(String testRoot, - FileEntry[] srcEntries, - FileEntry[] dstEntries, - boolean createSrcDir, - boolean createTgtDir, - boolean update) throws Exception { + FileEntry[] srcEntries, + FileEntry[] dstEntries, + boolean createSrcDir, + boolean createTgtDir, + boolean update) throws Exception { final String testSrcRel = SRCDAT; final String testSrc = testRoot + "/" + testSrcRel; final String testDstRel = DSTDAT; final String testDst = testRoot + "/" + testDstRel; String nnUri = FileSystem.getDefaultUri(conf).toString(); - DistributedFileSystem fs = (DistributedFileSystem) - FileSystem.get(URI.create(nnUri), conf); + FileSystem fs = FileSystem.get(URI.create(nnUri), conf); fs.mkdirs(new Path(testRoot)); if (createSrcDir) { fs.mkdirs(new Path(testSrc)); @@ -193,8 +129,8 @@ public class TestDistCpSystem { for(int i = 0; i < srcEntries.length; i++) { fs.setOwner(srcstats[i].getPath(), "u" + i, null); } - String[] args = update? new String[]{"-pub", "-update", nnUri+testSrc, - nnUri+testDst} : new String[]{"-pub", nnUri+testSrc, nnUri+testDst}; + String[] args = update? new String[]{"-pu", "-update", nnUri+testSrc, + nnUri+testDst} : new String[]{"-pu", nnUri+testSrc, nnUri+testDst}; ToolRunner.run(conf, new DistCp(), args); @@ -209,263 +145,20 @@ public class TestDistCpSystem { deldir(fs, testRoot); } - private void compareFiles(FileSystem fs, FileStatus srcStat, - FileStatus dstStat) throws Exception { - LOG.info("Comparing " + srcStat + " and " + dstStat); - assertEquals(srcStat.isDirectory(), dstStat.isDirectory()); - assertEquals(srcStat.getReplication(), dstStat.getReplication()); - assertEquals("File POSIX permission should match", - srcStat.getPermission(), dstStat.getPermission()); - assertEquals("File user ownership should match", - srcStat.getOwner(), dstStat.getOwner()); - assertEquals("File group ownership should match", - srcStat.getGroup(), dstStat.getGroup()); - // TODO; check ACL attributes - - if (srcStat.isDirectory()) { - return; - } - - assertEquals("File length should match (" + srcStat.getPath() + ")", - srcStat.getLen(), dstStat.getLen()); - - FSDataInputStream srcIn = fs.open(srcStat.getPath()); - FSDataInputStream dstIn = fs.open(dstStat.getPath()); - try { - byte[] readSrc = new byte[(int) - HdfsClientConfigKeys.DFS_BLOCK_SIZE_DEFAULT]; - byte[] readDst = new byte[(int) - HdfsClientConfigKeys.DFS_BLOCK_SIZE_DEFAULT]; - - int srcBytesRead = 0, tgtBytesRead = 0; - int srcIdx = 0, tgtIdx = 0; - long totalComparedBytes = 0; - while (true) { - if (srcBytesRead == 0) { - srcBytesRead = srcIn.read(readSrc); - srcIdx = 0; - } - if (tgtBytesRead == 0) { - tgtBytesRead = dstIn.read(readDst); - tgtIdx = 0; - } - if (srcBytesRead == 0 || tgtBytesRead == 0) { - LOG.info("______ compared src and dst files for " - + totalComparedBytes + " bytes, content match."); - if (srcBytesRead != tgtBytesRead) { - Assert.fail("Read mismatching size, compared " - + totalComparedBytes + " bytes between src and dst file " - + srcStat + " and " + dstStat); - } - if (totalComparedBytes != srcStat.getLen()) { - Assert.fail("Only read/compared " + totalComparedBytes + - " bytes between src and dst file " + srcStat + - " and " + dstStat); - } else { - // success - break; - } - } - for (; srcIdx < srcBytesRead && tgtIdx < tgtBytesRead; - ++srcIdx, ++tgtIdx) { - if (readSrc[srcIdx] != readDst[tgtIdx]) { - Assert.fail("src and dst file does not match at " - + totalComparedBytes + " between " - + srcStat + " and " + dstStat); - } - ++totalComparedBytes; - } - LOG.info("______ compared src and dst files for " - + totalComparedBytes + " bytes, content match. FileLength: " - + srcStat.getLen()); - if (totalComparedBytes == srcStat.getLen()) { - LOG.info("______ Final:" + srcIdx + " " - + srcBytesRead + " " + tgtIdx + " " + tgtBytesRead); - break; - } - if (srcIdx == srcBytesRead) { - srcBytesRead = 0; - } - if (tgtIdx == tgtBytesRead) { - tgtBytesRead = 0; - } - } - } finally { - if (srcIn != null) { - srcIn.close(); - } - if (dstIn != null) { - dstIn.close(); - } - } + @BeforeClass + public static void beforeClass() throws IOException { + conf = new Configuration(); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); + cluster.waitActive(); } - // WC: needed because the current distcp does not create target dirs - private void createDestDir(FileSystem fs, String testDst, - FileStatus[] srcStats, FileEntry[] srcFiles) throws IOException { - fs.mkdirs(new Path(testDst)); - - for (int i=0; i=0; --i) { - if (!srcFiles[i].isDirectory()) { - LOG.info("Modifying " + srcStats[i].getPath()); - DFSTestUtil.appendFileNewBlock(fs, srcStats[i].getPath(), - (int)BLOCK_SIZE * 3); - break; - } - } - // get file status after modifying file - srcStats = getFileStatus(fs, testRoot, srcFiles); - - args = new String[] {"-pugp", "-update", "-blocksperchunk", - String.valueOf(chunkSize), - nnUri + testSrc, nnUri + testDst + "/" + testSrcRel}; - - copyAndVerify(fs, srcFiles, srcStats, testDst, args); - - deldir(fs, testRoot); - } - - @Test - public void testRecursiveChunkCopy() throws Exception { - FileEntry[] srcFiles = { - new FileEntry(SRCDAT, true), - new FileEntry(SRCDAT + "/file0", false), - new FileEntry(SRCDAT + "/dir1", true), - new FileEntry(SRCDAT + "/dir2", true), - new FileEntry(SRCDAT + "/dir1/file1", false) - }; - chunkCopy(srcFiles); - } - - @Test - public void testChunkCopyOneFile() throws Exception { - FileEntry[] srcFiles = { - new FileEntry(SRCDAT, true), - new FileEntry(SRCDAT + "/file0", false) - }; - chunkCopy(srcFiles); - } - - @Test - public void testDistcpLargeFile() throws Exception { - FileEntry[] srcfiles = { - new FileEntry(SRCDAT, true), - new FileEntry(SRCDAT + "/file", false) - }; - - final String testRoot = "/testdir"; - final String testSrcRel = SRCDAT; - final String testSrc = testRoot + "/" + testSrcRel; - final String testDstRel = DSTDAT; - final String testDst = testRoot + "/" + testDstRel; - - String nnUri = FileSystem.getDefaultUri(conf).toString(); - DistributedFileSystem fs = - (DistributedFileSystem) FileSystem.get(URI.create(nnUri), conf); - fs.mkdirs(new Path(testRoot)); - fs.mkdirs(new Path(testSrc)); - fs.mkdirs(new Path(testDst)); - long chunkSize = 6; - createFiles(fs, testRoot, srcfiles, chunkSize); - - String srcFileName = testRoot + Path.SEPARATOR + srcfiles[1].getPath(); - Path srcfile = new Path(srcFileName); - - if(!cluster.getFileSystem().exists(srcfile)){ - throw new Exception("src not exist"); - } - - final long srcLen = fs.getFileStatus(srcfile).getLen(); - - FileStatus[] srcstats = getFileStatus(fs, testRoot, srcfiles); - for (int i = 0; i < srcfiles.length; i++) { - fs.setOwner(srcstats[i].getPath(), "u" + i, null); - } - String[] args = new String[] { - "-blocksperchunk", - String.valueOf(chunkSize), - nnUri + testSrc, - nnUri + testDst - }; - - LOG.info("_____ running distcp: " + args[0] + " " + args[1]); - ToolRunner.run(conf, new DistCp(), args); - - String realTgtPath = testDst; - FileStatus[] dststat = getFileStatus(fs, realTgtPath, srcfiles); - assertEquals("File length should match", srcLen, - dststat[dststat.length - 1].getLen()); - - this.compareFiles(fs, srcstats[srcstats.length-1], - dststat[dststat.length-1]); - deldir(fs, testRoot); - } - @Test public void testPreserveUseNonEmptyDir() throws Exception { String testRoot = "/testdir." + getMethodName(); @@ -487,6 +180,7 @@ public class TestDistCpSystem { testPreserveUserHelper(testRoot, srcfiles, dstfiles, false, false, false); } + @Test public void testPreserveUserEmptyDir() throws Exception { String testRoot = "/testdir." + getMethodName(); diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java index f94ba97ec34..efe46272e38 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java @@ -394,7 +394,7 @@ public class TestOptionsParser { + "copyStrategy='uniformsize', preserveStatus=[], " + "preserveRawXattrs=false, atomicWorkPath=null, logPath=null, " + "sourceFileListing=abc, sourcePaths=null, targetPath=xyz, " - + "targetPathExists=true, filtersFile='null', blocksPerChunk=0}"; + + "targetPathExists=true, filtersFile='null'}"; String optionString = option.toString(); Assert.assertEquals(val, optionString); Assert.assertNotSame(DistCpOptionSwitch.ATOMIC_COMMIT.toString(), diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java index 2452d6fee3d..2e9a350b1ca 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java @@ -81,10 +81,6 @@ public class TestCopyCommitter { @Before public void createMetaFolder() { config.set(DistCpConstants.CONF_LABEL_META_FOLDER, "/meta"); - // Unset listing file path since the config is shared by - // multiple tests, and some test doesn't set it, such as - // testNoCommitAction, but the distcp code will check it. - config.set(DistCpConstants.CONF_LABEL_LISTING_FILE_PATH, ""); Path meta = new Path("/meta"); try { cluster.getFileSystem().mkdirs(meta); @@ -330,6 +326,7 @@ public class TestCopyCommitter { committer.commitJob(jobContext); Assert.assertFalse(fs.exists(new Path(workPath))); Assert.assertTrue(fs.exists(new Path(finalPath))); + } catch (IOException e) { LOG.error("Exception encountered while testing for preserve status", e); Assert.fail("Atomic commit failure"); From bf3fb585aaf2b179836e139c041fc87920a3c886 Mon Sep 17 00:00:00 2001 From: Yongjun Zhang Date: Thu, 30 Mar 2017 17:01:15 -0700 Subject: [PATCH 173/188] HADOOP-11794. Enable distcp to copy blocks in parallel. Contributed by Yongjun Zhang, Wei-Chiu Chuang, Xiao Chen, Rosie Li. --- .../org/apache/hadoop/hdfs/DFSTestUtil.java | 22 +- .../org/apache/hadoop/tools/CopyListing.java | 37 +- .../hadoop/tools/CopyListingFileStatus.java | 87 ++++- .../java/org/apache/hadoop/tools/DistCp.java | 52 +++ .../hadoop/tools/DistCpOptionSwitch.java | 10 + .../apache/hadoop/tools/DistCpOptions.java | 22 +- .../apache/hadoop/tools/OptionsParser.java | 36 +- .../hadoop/tools/SimpleCopyListing.java | 83 ++-- .../hadoop/tools/mapred/CopyCommitter.java | 174 ++++++++- .../hadoop/tools/mapred/CopyMapper.java | 40 +- .../mapred/RetriableFileCopyCommand.java | 26 +- .../tools/mapred/UniformSizeInputFormat.java | 5 +- .../apache/hadoop/tools/util/DistCpUtils.java | 111 +++++- .../src/site/markdown/DistCp.md.vm | 1 + .../apache/hadoop/tools/TestDistCpSystem.java | 368 ++++++++++++++++-- .../hadoop/tools/TestOptionsParser.java | 2 +- .../tools/mapred/TestCopyCommitter.java | 5 +- 17 files changed, 971 insertions(+), 110 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java index 13291952a9e..9b782f32f2a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java @@ -862,7 +862,27 @@ public class DFSTestUtil { out.write(toAppend); } } - + + /** + * Append specified length of bytes to a given file, starting with new block. + * @param fs The file system + * @param p Path of the file to append + * @param length Length of bytes to append to the file + * @throws IOException + */ + public static void appendFileNewBlock(DistributedFileSystem fs, + Path p, int length) throws IOException { + assert fs.exists(p); + assert length >= 0; + byte[] toAppend = new byte[length]; + Random random = new Random(); + random.nextBytes(toAppend); + try (FSDataOutputStream out = fs.append(p, + EnumSet.of(CreateFlag.APPEND, CreateFlag.NEW_BLOCK), 4096, null)) { + out.write(toAppend); + } + } + /** * @return url content as string (UTF-8 encoding assumed) */ diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java index 481aa61b0f1..9ebf9d29b3b 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java @@ -145,12 +145,22 @@ public abstract class CopyListing extends Configured { Configuration config = getConf(); FileSystem fs = pathToListFile.getFileSystem(config); - Path sortedList = DistCpUtils.sortListing(fs, config, pathToListFile); + final boolean splitLargeFile = options.splitLargeFile(); + + // When splitLargeFile is enabled, we don't randomize the copylist + // earlier, so we don't do the sorting here. For a file that has + // multiple entries due to split, we check here that their + // is continuous. + // + Path checkPath = splitLargeFile? + pathToListFile : DistCpUtils.sortListing(fs, config, pathToListFile); SequenceFile.Reader reader = new SequenceFile.Reader( - config, SequenceFile.Reader.file(sortedList)); + config, SequenceFile.Reader.file(checkPath)); try { Text lastKey = new Text("*"); //source relative path can never hold * + long lastChunkOffset = -1; + long lastChunkLength = -1; CopyListingFileStatus lastFileStatus = new CopyListingFileStatus(); Text currentKey = new Text(); @@ -161,8 +171,21 @@ public abstract class CopyListing extends Configured { if (currentKey.equals(lastKey)) { CopyListingFileStatus currentFileStatus = new CopyListingFileStatus(); reader.getCurrentValue(currentFileStatus); - throw new DuplicateFileException("File " + lastFileStatus.getPath() + " and " + - currentFileStatus.getPath() + " would cause duplicates. Aborting"); + if (!splitLargeFile) { + throw new DuplicateFileException("File " + lastFileStatus.getPath() + + " and " + currentFileStatus.getPath() + + " would cause duplicates. Aborting"); + } else { + if (lastChunkOffset + lastChunkLength != + currentFileStatus.getChunkOffset()) { + throw new InvalidInputException("File " + lastFileStatus.getPath() + + " " + lastChunkOffset + "," + lastChunkLength + + " and " + currentFileStatus.getPath() + + " " + currentFileStatus.getChunkOffset() + "," + + currentFileStatus.getChunkLength() + + " are not continuous. Aborting"); + } + } } reader.getCurrentValue(lastFileStatus); if (options.shouldPreserve(DistCpOptions.FileAttribute.ACL)) { @@ -181,8 +204,12 @@ public abstract class CopyListing extends Configured { xAttrSupportCheckFsSet.add(lastFsUri); } } - lastKey.set(currentKey); + lastKey.set(currentKey); + if (splitLargeFile) { + lastChunkOffset = lastFileStatus.getChunkOffset(); + lastChunkLength = lastFileStatus.getChunkLength(); + } if (options.shouldUseDiff() && LOG.isDebugEnabled()) { LOG.debug("Copy list entry " + idx + ": " + lastFileStatus.getPath().toUri().getPath()); diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java index 00d4b325053..29c59ac1033 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java @@ -74,6 +74,14 @@ public final class CopyListingFileStatus implements Writable { private List aclEntries; private Map xAttrs; + // represents the offset and length of a file + // chunk in number of bytes. + // used when splitting a large file to chunks to copy in parallel. + // If a file is not large enough to split, chunkOffset would be 0 and + // chunkLength would be the length of the file. + private long chunkOffset = 0; + private long chunkLength = Long.MAX_VALUE; + /** * Default constructor. */ @@ -96,11 +104,32 @@ public final class CopyListingFileStatus implements Writable { fileStatus.getPath()); } + public CopyListingFileStatus(FileStatus fileStatus, + long chunkOffset, long chunkLength) { + this(fileStatus.getLen(), fileStatus.isDirectory(), + fileStatus.getReplication(), fileStatus.getBlockSize(), + fileStatus.getModificationTime(), fileStatus.getAccessTime(), + fileStatus.getPermission(), fileStatus.getOwner(), + fileStatus.getGroup(), + fileStatus.getPath()); + this.chunkOffset = chunkOffset; + this.chunkLength = chunkLength; + } + @SuppressWarnings("checkstyle:parameternumber") public CopyListingFileStatus(long length, boolean isdir, int blockReplication, long blocksize, long modificationTime, long accessTime, FsPermission permission, String owner, String group, Path path) { + this(length, isdir, blockReplication, blocksize, modificationTime, + accessTime, permission, owner, group, path, 0, Long.MAX_VALUE); + } + + @SuppressWarnings("checkstyle:parameternumber") + public CopyListingFileStatus(long length, boolean isdir, + int blockReplication, long blocksize, long modificationTime, + long accessTime, FsPermission permission, String owner, String group, + Path path, long chunkOffset, long chunkLength) { this.length = length; this.isdir = isdir; this.blockReplication = (short)blockReplication; @@ -117,6 +146,23 @@ public final class CopyListingFileStatus implements Writable { this.owner = (owner == null) ? "" : owner; this.group = (group == null) ? "" : group; this.path = path; + this.chunkOffset = chunkOffset; + this.chunkLength = chunkLength; + } + + public CopyListingFileStatus(CopyListingFileStatus other) { + this.length = other.length; + this.isdir = other.isdir; + this.blockReplication = other.blockReplication; + this.blocksize = other.blocksize; + this.modificationTime = other.modificationTime; + this.accessTime = other.accessTime; + this.permission = other.permission; + this.owner = other.owner; + this.group = other.group; + this.path = new Path(other.path.toUri()); + this.chunkOffset = other.chunkOffset; + this.chunkLength = other.chunkLength; } public Path getPath() { @@ -200,6 +246,31 @@ public final class CopyListingFileStatus implements Writable { this.xAttrs = xAttrs; } + public long getChunkOffset() { + return chunkOffset; + } + + public void setChunkOffset(long offset) { + this.chunkOffset = offset; + } + + public long getChunkLength() { + return chunkLength; + } + + public void setChunkLength(long chunkLength) { + this.chunkLength = chunkLength; + } + + public boolean isSplit() { + return getChunkLength() != Long.MAX_VALUE && + getChunkLength() != getLen(); + } + + public long getSizeToCopy() { + return isSplit()? getChunkLength() : getLen(); + } + @Override public void write(DataOutput out) throws IOException { Text.writeString(out, getPath().toString(), Text.DEFAULT_MAX_LEN); @@ -244,6 +315,9 @@ public final class CopyListingFileStatus implements Writable { } else { out.writeInt(NO_XATTRS); } + + out.writeLong(chunkOffset); + out.writeLong(chunkLength); } @Override @@ -292,6 +366,9 @@ public final class CopyListingFileStatus implements Writable { } else { xAttrs = null; } + + chunkOffset = in.readLong(); + chunkLength = in.readLong(); } @Override @@ -317,8 +394,14 @@ public final class CopyListingFileStatus implements Writable { public String toString() { StringBuilder sb = new StringBuilder(super.toString()); sb.append('{'); - sb.append("aclEntries = " + aclEntries); - sb.append(", xAttrs = " + xAttrs); + sb.append(this.getPath().toString()); + sb.append(" length = ").append(this.getLen()); + sb.append(" aclEntries = ").append(aclEntries); + sb.append(", xAttrs = ").append(xAttrs); + if (isSplit()) { + sb.append(", chunkOffset = ").append(this.getChunkOffset()); + sb.append(", chunkLength = ").append(this.getChunkLength()); + } sb.append('}'); return sb.toString(); } diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java index ab58e9c66d8..8c2fa24a15c 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java @@ -35,6 +35,7 @@ import org.apache.hadoop.mapreduce.Cluster; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.JobSubmissionFiles; +import org.apache.hadoop.tools.DistCpOptions.FileAttribute; import org.apache.hadoop.tools.CopyListing.*; import org.apache.hadoop.tools.mapred.CopyMapper; import org.apache.hadoop.tools.mapred.CopyOutputFormat; @@ -134,6 +135,7 @@ public class DistCp extends Configured implements Tool { try { inputOptions = (OptionsParser.parse(argv)); + setOptionsForSplitLargeFile(); setTargetPathExists(); LOG.info("Input Options: " + inputOptions); } catch (Throwable e) { @@ -235,6 +237,56 @@ public class DistCp extends Configured implements Tool { getConf().setBoolean(DistCpConstants.CONF_LABEL_TARGET_PATH_EXISTS, targetExists); } + + /** + * Check if concat is supported by fs. + * Throws UnsupportedOperationException if not. + */ + private void checkConcatSupport(FileSystem fs) { + try { + Path[] src = null; + Path tgt = null; + fs.concat(tgt, src); + } catch (UnsupportedOperationException use) { + throw new UnsupportedOperationException( + DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch() + + " is not supported since the target file system doesn't" + + " support concat.", use); + } catch (Exception e) { + // Ignore other exception + } + } + + /** + * Set up needed options for splitting large files. + */ + private void setOptionsForSplitLargeFile() throws IOException { + if (!inputOptions.splitLargeFile()) { + return; + } + Path target = inputOptions.getTargetPath(); + FileSystem targetFS = target.getFileSystem(getConf()); + checkConcatSupport(targetFS); + + LOG.info("Enabling preserving blocksize since " + + DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch() + " is passed."); + inputOptions.preserve(FileAttribute.BLOCKSIZE); + + LOG.info("Set " + + DistCpOptionSwitch.APPEND.getSwitch() + + " to false since " + DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch() + + " is passed."); + inputOptions.setAppend(false); + + LOG.info("Set " + + DistCpConstants.CONF_LABEL_SIMPLE_LISTING_RANDOMIZE_FILES + + " to false since " + DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch() + + " is passed."); + getConf().setBoolean( + DistCpConstants.CONF_LABEL_SIMPLE_LISTING_RANDOMIZE_FILES, false); + } + + /** * Create Job object for submitting it, with all the configuration * diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java index fb47d769233..ced9b540c89 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java @@ -169,6 +169,16 @@ public enum DistCpOptionSwitch { new Option("sizelimit", true, "(Deprecated!) Limit number of files " + "copied to <= n bytes")), + BLOCKS_PER_CHUNK("", + new Option("blocksperchunk", true, "If set to a positive value, files" + + "with more blocks than this value will be split into chunks of " + + " blocks to be transferred in parallel, and " + + "reassembled on the destination. By default, is " + + "0 and the files will be transmitted in their entirety without " + + "splitting. This switch is only applicable when the source file " + + "system implements getBlockLocations method and the target file " + + "system implements concat method")), + /** * Specify bandwidth per map in MB, accepts bandwidth as a fraction */ diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java index 8c37ff30ae0..9822d83dbd0 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java @@ -97,7 +97,11 @@ public class DistCpOptions { // targetPathExist is a derived field, it's initialized in the // beginning of distcp. private boolean targetPathExists = true; - + + // Size of chunk in number of blocks when splitting large file into chunks + // to copy in parallel. Default is 0 and file are not splitted. + private int blocksPerChunk = 0; + public static enum FileAttribute{ REPLICATION, BLOCKSIZE, USER, GROUP, PERMISSION, CHECKSUMTYPE, ACL, XATTR, TIMES; @@ -166,6 +170,7 @@ public class DistCpOptions { this.targetPath = that.getTargetPath(); this.targetPathExists = that.getTargetPathExists(); this.filtersFile = that.getFiltersFile(); + this.blocksPerChunk = that.blocksPerChunk; } } @@ -578,6 +583,18 @@ public class DistCpOptions { this.filtersFile = filtersFilename; } + public final void setBlocksPerChunk(int csize) { + this.blocksPerChunk = csize; + } + + public final int getBlocksPerChunk() { + return blocksPerChunk; + } + + public final boolean splitLargeFile() { + return blocksPerChunk > 0; + } + void validate() { if ((useDiff || useRdiff) && deleteMissing) { // -delete and -diff/-rdiff are mutually exclusive. For backward @@ -669,6 +686,8 @@ public class DistCpOptions { DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.FILTERS, filtersFile); } + DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.BLOCKS_PER_CHUNK, + String.valueOf(blocksPerChunk)); } /** @@ -704,6 +723,7 @@ public class DistCpOptions { ", targetPath=" + targetPath + ", targetPathExists=" + targetPathExists + ", filtersFile='" + filtersFile + '\'' + + ", blocksPerChunk=" + blocksPerChunk + '}'; } diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java index d0f82ca76b5..8881264d281 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java @@ -172,11 +172,44 @@ public class OptionsParser { DistCpOptionSwitch.FILTERS.getSwitch())); } + parseBlocksPerChunk(command, option); + option.validate(); return option; } + + /** + * A helper method to parse chunk size in number of blocks. + * Used when breaking large file into chunks to copy in parallel. + * + * @param command command line arguments + */ + private static void parseBlocksPerChunk(CommandLine command, + DistCpOptions option) { + boolean hasOption = + command.hasOption(DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch()); + LOG.info("parseChunkSize: " + + DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch() + " " + hasOption); + if (hasOption) { + String chunkSizeString = getVal(command, + DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch().trim()); + try { + int csize = Integer.parseInt(chunkSizeString); + if (csize < 0) { + csize = 0; + } + LOG.info("Set distcp blocksPerChunk to " + csize); + option.setBlocksPerChunk(csize); + } + catch (NumberFormatException e) { + throw new IllegalArgumentException("blocksPerChunk is invalid: " + + chunkSizeString, e); + } + } + } + /** * parseSizeLimit is a helper method for parsing the deprecated * argument SIZE_LIMIT. @@ -211,8 +244,7 @@ public class OptionsParser { DistCpOptionSwitch.FILE_LIMIT.getSwitch().trim()); try { Integer.parseInt(fileLimitString); - } - catch (NumberFormatException e) { + } catch (NumberFormatException e) { throw new IllegalArgumentException("File-limit is invalid: " + fileLimitString, e); } diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java index 105e4f2fe17..af913474550 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java @@ -19,6 +19,7 @@ package org.apache.hadoop.tools; import com.google.common.collect.Lists; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; @@ -47,6 +48,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Random; +import java.util.LinkedList; import static org.apache.hadoop.tools.DistCpConstants .HDFS_RESERVED_RAW_DIRECTORY_NAME; @@ -240,10 +242,10 @@ public class SimpleCopyListing extends CopyListing { final boolean preserveAcls = options.shouldPreserve(FileAttribute.ACL); final boolean preserveXAttrs = options.shouldPreserve(FileAttribute.XATTR); final boolean preserveRawXAttrs = options.shouldPreserveRawXattrs(); - CopyListingFileStatus fileCopyListingStatus = + LinkedList fileCopyListingStatus = DistCpUtils.toCopyListingFileStatus(sourceFS, fileStatus, - preserveAcls, preserveXAttrs, preserveRawXAttrs); - + preserveAcls, preserveXAttrs, preserveRawXAttrs, + options.getBlocksPerChunk()); writeToFileListingRoot(fileListWriter, fileCopyListingStatus, sourceRoot, options); } @@ -348,9 +350,10 @@ public class SimpleCopyListing extends CopyListing { FileStatus[] sourceFiles = sourceFS.listStatus(path); boolean explore = (sourceFiles != null && sourceFiles.length > 0); if (!explore || rootStatus.isDirectory()) { - CopyListingFileStatus rootCopyListingStatus = - DistCpUtils.toCopyListingFileStatus(sourceFS, rootStatus, - preserveAcls, preserveXAttrs, preserveRawXAttrs); + LinkedList rootCopyListingStatus = + DistCpUtils.toCopyListingFileStatus(sourceFS, rootStatus, + preserveAcls, preserveXAttrs, preserveRawXAttrs, + options.getBlocksPerChunk()); writeToFileListingRoot(fileListWriter, rootCopyListingStatus, sourcePathRoot, options); } @@ -360,20 +363,20 @@ public class SimpleCopyListing extends CopyListing { if (LOG.isDebugEnabled()) { LOG.debug("Recording source-path: " + sourceStatus.getPath() + " for copy."); } - CopyListingFileStatus sourceCopyListingStatus = - DistCpUtils.toCopyListingFileStatus(sourceFS, sourceStatus, - preserveAcls && sourceStatus.isDirectory(), - preserveXAttrs && sourceStatus.isDirectory(), - preserveRawXAttrs && sourceStatus.isDirectory()); - if (randomizeFileListing) { - addToFileListing(statusList, - new FileStatusInfo(sourceCopyListingStatus, sourcePathRoot), - fileListWriter); - } else { - writeToFileListing(fileListWriter, sourceCopyListingStatus, - sourcePathRoot); + LinkedList sourceCopyListingStatus = + DistCpUtils.toCopyListingFileStatus(sourceFS, sourceStatus, + preserveAcls && sourceStatus.isDirectory(), + preserveXAttrs && sourceStatus.isDirectory(), + preserveRawXAttrs && sourceStatus.isDirectory(), + options.getBlocksPerChunk()); + for (CopyListingFileStatus fs : sourceCopyListingStatus) { + if (randomizeFileListing) { + addToFileListing(statusList, + new FileStatusInfo(fs, sourcePathRoot), fileListWriter); + } else { + writeToFileListing(fileListWriter, fs, sourcePathRoot); + } } - if (sourceStatus.isDirectory()) { if (LOG.isDebugEnabled()) { LOG.debug("Adding source dir for traverse: " + sourceStatus.getPath()); @@ -641,18 +644,20 @@ public class SimpleCopyListing extends CopyListing { LOG.debug("Recording source-path: " + child.getPath() + " for copy."); } if (workResult.getSuccess()) { - CopyListingFileStatus childCopyListingStatus = + LinkedList childCopyListingStatus = DistCpUtils.toCopyListingFileStatus(sourceFS, child, preserveAcls && child.isDirectory(), preserveXAttrs && child.isDirectory(), - preserveRawXattrs && child.isDirectory()); - if (randomizeFileListing) { - addToFileListing(fileStatuses, - new FileStatusInfo(childCopyListingStatus, sourcePathRoot), - fileListWriter); - } else { - writeToFileListing(fileListWriter, childCopyListingStatus, - sourcePathRoot); + preserveRawXattrs && child.isDirectory(), + options.getBlocksPerChunk()); + + for (CopyListingFileStatus fs : childCopyListingStatus) { + if (randomizeFileListing) { + addToFileListing(fileStatuses, + new FileStatusInfo(fs, sourcePathRoot), fileListWriter); + } else { + writeToFileListing(fileListWriter, fs, sourcePathRoot); + } } } if (retry < maxRetries) { @@ -675,19 +680,21 @@ public class SimpleCopyListing extends CopyListing { } private void writeToFileListingRoot(SequenceFile.Writer fileListWriter, - CopyListingFileStatus fileStatus, Path sourcePathRoot, + LinkedList fileStatus, Path sourcePathRoot, DistCpOptions options) throws IOException { boolean syncOrOverwrite = options.shouldSyncFolder() || options.shouldOverwrite(); - if (fileStatus.getPath().equals(sourcePathRoot) && - fileStatus.isDirectory() && syncOrOverwrite) { - // Skip the root-paths when syncOrOverwrite - if (LOG.isDebugEnabled()) { - LOG.debug("Skip " + fileStatus.getPath()); - } - return; + for (CopyListingFileStatus fs : fileStatus) { + if (fs.getPath().equals(sourcePathRoot) && + fs.isDirectory() && syncOrOverwrite) { + // Skip the root-paths when syncOrOverwrite + if (LOG.isDebugEnabled()) { + LOG.debug("Skip " + fs.getPath()); + } + return; + } + writeToFileListing(fileListWriter, fs, sourcePathRoot); } - writeToFileListing(fileListWriter, fileStatus, sourcePathRoot); } private void writeToFileListing(SequenceFile.Writer fileListWriter, @@ -707,7 +714,7 @@ public class SimpleCopyListing extends CopyListing { fileListWriter.sync(); if (!fileStatus.isDirectory()) { - totalBytesToCopy += fileStatus.getLen(); + totalBytesToCopy += fileStatus.getSizeToCopy(); } else { totalDirs++; } diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java index 75cefb488ae..6ddaab99c3d 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java @@ -27,6 +27,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.Text; +import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.JobStatus; import org.apache.hadoop.mapreduce.TaskAttemptContext; @@ -34,14 +35,17 @@ import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter; import org.apache.hadoop.tools.CopyListing; import org.apache.hadoop.tools.CopyListingFileStatus; import org.apache.hadoop.tools.DistCpConstants; +import org.apache.hadoop.tools.DistCpOptionSwitch; import org.apache.hadoop.tools.DistCpOptions; import org.apache.hadoop.tools.DistCpOptions.FileAttribute; import org.apache.hadoop.tools.GlobbedCopyListing; import org.apache.hadoop.tools.util.DistCpUtils; +import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; import java.util.EnumSet; +import java.util.LinkedList; import java.util.List; /** @@ -63,7 +67,8 @@ public class CopyCommitter extends FileOutputCommitter { private boolean syncFolder = false; private boolean overwrite = false; private boolean targetPathExists = true; - + private boolean ignoreFailures = false; + /** * Create a output committer * @@ -82,8 +87,13 @@ public class CopyCommitter extends FileOutputCommitter { Configuration conf = jobContext.getConfiguration(); syncFolder = conf.getBoolean(DistCpConstants.CONF_LABEL_SYNC_FOLDERS, false); overwrite = conf.getBoolean(DistCpConstants.CONF_LABEL_OVERWRITE, false); - targetPathExists = conf.getBoolean(DistCpConstants.CONF_LABEL_TARGET_PATH_EXISTS, true); - + targetPathExists = conf.getBoolean( + DistCpConstants.CONF_LABEL_TARGET_PATH_EXISTS, true); + ignoreFailures = conf.getBoolean( + DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false); + + concatFileChunks(conf); + super.commitJob(jobContext); cleanupTempFiles(jobContext); @@ -169,9 +179,112 @@ public class CopyCommitter extends FileOutputCommitter { } } + private boolean isFileNotFoundException(IOException e) { + if (e instanceof FileNotFoundException) { + return true; + } + + if (e instanceof RemoteException) { + return ((RemoteException)e).unwrapRemoteException() + instanceof FileNotFoundException; + } + + return false; + } + + /** + * Concat chunk files for the same file into one. + * Iterate through copy listing, identify chunk files for the same file, + * concat them into one. + */ + private void concatFileChunks(Configuration conf) throws IOException { + + LOG.info("concat file chunks ..."); + + String spath = conf.get(DistCpConstants.CONF_LABEL_LISTING_FILE_PATH); + if (spath == null || spath.isEmpty()) { + return; + } + Path sourceListing = new Path(spath); + SequenceFile.Reader sourceReader = new SequenceFile.Reader(conf, + SequenceFile.Reader.file(sourceListing)); + Path targetRoot = + new Path(conf.get(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH)); + + try { + CopyListingFileStatus srcFileStatus = new CopyListingFileStatus(); + Text srcRelPath = new Text(); + CopyListingFileStatus lastFileStatus = null; + LinkedList allChunkPaths = new LinkedList(); + + // Iterate over every source path that was copied. + while (sourceReader.next(srcRelPath, srcFileStatus)) { + if (srcFileStatus.isDirectory()) { + continue; + } + Path targetFile = new Path(targetRoot.toString() + "/" + srcRelPath); + Path targetFileChunkPath = + DistCpUtils.getSplitChunkPath(targetFile, srcFileStatus); + if (LOG.isDebugEnabled()) { + LOG.debug(" add " + targetFileChunkPath + " to concat."); + } + allChunkPaths.add(targetFileChunkPath); + if (srcFileStatus.getChunkOffset() + srcFileStatus.getChunkLength() + == srcFileStatus.getLen()) { + // This is the last chunk of the splits, consolidate allChunkPaths + try { + concatFileChunks(conf, targetFile, allChunkPaths); + } catch (IOException e) { + // If the concat failed because a chunk file doesn't exist, + // then we assume that the CopyMapper has skipped copying this + // file, and we ignore the exception here. + // If a chunk file should have been created but it was not, then + // the CopyMapper would have failed. + if (!isFileNotFoundException(e)) { + String emsg = "Failed to concat chunk files for " + targetFile; + if (!ignoreFailures) { + throw new IOException(emsg, e); + } else { + LOG.warn(emsg, e); + } + } + } + allChunkPaths.clear(); + lastFileStatus = null; + } else { + if (lastFileStatus == null) { + lastFileStatus = new CopyListingFileStatus(srcFileStatus); + } else { + // Two neighboring chunks have to be consecutive ones for the same + // file, for them to be merged + if (!srcFileStatus.getPath().equals(lastFileStatus.getPath()) || + srcFileStatus.getChunkOffset() != + (lastFileStatus.getChunkOffset() + + lastFileStatus.getChunkLength())) { + String emsg = "Inconsistent sequence file: current " + + "chunk file " + srcFileStatus + " doesnt match prior " + + "entry " + lastFileStatus; + if (!ignoreFailures) { + throw new IOException(emsg); + } else { + LOG.warn(emsg + ", skipping concat this set."); + } + } else { + lastFileStatus.setChunkOffset(srcFileStatus.getChunkOffset()); + lastFileStatus.setChunkLength(srcFileStatus.getChunkLength()); + } + } + } + } + } finally { + IOUtils.closeStream(sourceReader); + } + } + // This method changes the target-directories' file-attributes (owner, // user/group permissions, etc.) based on the corresponding source directories. - private void preserveFileAttributesForDirectories(Configuration conf) throws IOException { + private void preserveFileAttributesForDirectories(Configuration conf) + throws IOException { String attrSymbols = conf.get(DistCpConstants.CONF_LABEL_PRESERVE_STATUS); final boolean syncOrOverwrite = syncFolder || overwrite; @@ -325,4 +438,57 @@ public class CopyCommitter extends FileOutputCommitter { ", Unable to move to " + finalDir); } } + + /** + * Concat the passed chunk files into one and rename it the targetFile. + */ + private void concatFileChunks(Configuration conf, Path targetFile, + LinkedList allChunkPaths) throws IOException { + if (allChunkPaths.size() == 1) { + return; + } + if (LOG.isDebugEnabled()) { + LOG.debug("concat " + targetFile + " allChunkSize+ " + + allChunkPaths.size()); + } + FileSystem dstfs = targetFile.getFileSystem(conf); + + Path firstChunkFile = allChunkPaths.removeFirst(); + Path[] restChunkFiles = new Path[allChunkPaths.size()]; + allChunkPaths.toArray(restChunkFiles); + if (LOG.isDebugEnabled()) { + LOG.debug("concat: firstchunk: " + dstfs.getFileStatus(firstChunkFile)); + int i = 0; + for (Path f : restChunkFiles) { + LOG.debug("concat: other chunk: " + i + ": " + dstfs.getFileStatus(f)); + ++i; + } + } + dstfs.concat(firstChunkFile, restChunkFiles); + if (LOG.isDebugEnabled()) { + LOG.debug("concat: result: " + dstfs.getFileStatus(firstChunkFile)); + } + rename(dstfs, firstChunkFile, targetFile); + } + + /** + * Rename tmp to dst on destFileSys. + * @param destFileSys the file ssystem + * @param tmp the source path + * @param dst the destination path + * @throws IOException if renaming failed + */ + private static void rename(FileSystem destFileSys, Path tmp, Path dst) + throws IOException { + try { + if (destFileSys.exists(dst)) { + destFileSys.delete(dst, true); + } + destFileSys.rename(tmp, dst); + } catch (IOException ioe) { + throw new IOException("Fail to rename tmp file (=" + tmp + + ") to destination file (=" + dst + ")", ioe); + } + } + } diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java index e1873f17e41..d6b3ba817b0 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java @@ -156,10 +156,12 @@ public class CopyMapper extends Mapper sourceFS = sourcePath.getFileSystem(conf); final boolean preserveXAttrs = fileAttributes.contains(FileAttribute.XATTR); - sourceCurrStatus = DistCpUtils.toCopyListingFileStatus(sourceFS, - sourceFS.getFileStatus(sourcePath), - fileAttributes.contains(FileAttribute.ACL), - preserveXAttrs, preserveRawXattrs); + sourceCurrStatus = DistCpUtils.toCopyListingFileStatusHelper(sourceFS, + sourceFS.getFileStatus(sourcePath), + fileAttributes.contains(FileAttribute.ACL), + preserveXAttrs, preserveRawXattrs, + sourceFileStatus.getChunkOffset(), + sourceFileStatus.getChunkLength()); } catch (FileNotFoundException e) { throw new IOException(new RetriableFileCopyCommand.CopyReadException(e)); } @@ -173,7 +175,8 @@ public class CopyMapper extends Mapper LOG.debug("Path could not be found: " + target, ignore); } - if (targetStatus != null && (targetStatus.isDirectory() != sourceCurrStatus.isDirectory())) { + if (targetStatus != null && + (targetStatus.isDirectory() != sourceCurrStatus.isDirectory())) { throw new IOException("Can't replace " + target + ". Target is " + getFileType(targetStatus) + ", Source is " + getFileType(sourceCurrStatus)); } @@ -183,19 +186,28 @@ public class CopyMapper extends Mapper return; } - FileAction action = checkUpdate(sourceFS, sourceCurrStatus, target, targetStatus); + FileAction action = checkUpdate(sourceFS, sourceCurrStatus, target, + targetStatus); + + Path tmpTarget = target; if (action == FileAction.SKIP) { LOG.info("Skipping copy of " + sourceCurrStatus.getPath() + " to " + target); updateSkipCounters(context, sourceCurrStatus); context.write(null, new Text("SKIP: " + sourceCurrStatus.getPath())); + } else { - copyFileWithRetry(description, sourceCurrStatus, target, context, + if (sourceCurrStatus.isSplit()) { + tmpTarget = DistCpUtils.getSplitChunkPath(target, sourceCurrStatus); + } + if (LOG.isDebugEnabled()) { + LOG.debug("copying " + sourceCurrStatus + " " + tmpTarget); + } + copyFileWithRetry(description, sourceCurrStatus, tmpTarget, context, action, fileAttributes); } - - DistCpUtils.preserve(target.getFileSystem(conf), target, sourceCurrStatus, - fileAttributes, preserveRawXattrs); + DistCpUtils.preserve(target.getFileSystem(conf), tmpTarget, + sourceCurrStatus, fileAttributes, preserveRawXattrs); } catch (IOException exception) { handleFailures(exception, sourceFileStatus, target, context); } @@ -261,8 +273,12 @@ public class CopyMapper extends Mapper private void handleFailures(IOException exception, CopyListingFileStatus sourceFileStatus, Path target, Context context) throws IOException, InterruptedException { - LOG.error("Failure in copying " + sourceFileStatus.getPath() + " to " + - target, exception); + LOG.error("Failure in copying " + sourceFileStatus.getPath() + + (sourceFileStatus.isSplit()? "," + + " offset=" + sourceFileStatus.getChunkOffset() + + " chunkLength=" + sourceFileStatus.getChunkLength() + : "") + + " to " + target, exception); if (ignoreFailures && ExceptionUtils.indexOfType(exception, CopyReadException.class) != -1) { diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java index 06acd78a8a1..2c17fef1a38 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java @@ -118,17 +118,21 @@ public class RetriableFileCopyCommand extends RetriableCommand { .contains(FileAttribute.CHECKSUMTYPE) ? sourceFS .getFileChecksum(sourcePath) : null; - final long offset = action == FileAction.APPEND ? targetFS.getFileStatus( - target).getLen() : 0; + long offset = (action == FileAction.APPEND) ? + targetFS.getFileStatus(target).getLen() : source.getChunkOffset(); long bytesRead = copyToFile(targetPath, targetFS, source, offset, context, fileAttributes, sourceChecksum); - compareFileLengths(source, targetPath, configuration, bytesRead - + offset); + if (!source.isSplit()) { + compareFileLengths(source, targetPath, configuration, bytesRead + + offset); + } //At this point, src&dest lengths are same. if length==0, we skip checksum if ((bytesRead != 0) && (!skipCrc)) { - compareCheckSums(sourceFS, source.getPath(), sourceChecksum, - targetFS, targetPath); + if (!source.isSplit()) { + compareCheckSums(sourceFS, source.getPath(), sourceChecksum, + targetFS, targetPath); + } } // it's not append case, thus we first write to a temporary file, rename // it to the target path. @@ -249,16 +253,26 @@ public class RetriableFileCopyCommand extends RetriableCommand { ThrottledInputStream inStream = null; long totalBytesRead = 0; + long chunkLength = source2.getChunkLength(); + boolean finished = false; try { inStream = getInputStream(source, context.getConfiguration()); int bytesRead = readBytes(inStream, buf, sourceOffset); while (bytesRead >= 0) { + if (chunkLength > 0 && + (totalBytesRead + bytesRead) >= chunkLength) { + bytesRead = (int)(chunkLength - totalBytesRead); + finished = true; + } totalBytesRead += bytesRead; if (action == FileAction.APPEND) { sourceOffset += bytesRead; } outStream.write(buf, 0, bytesRead); updateContextStatus(totalBytesRead, context, source2); + if (finished) { + break; + } bytesRead = readBytes(inStream, buf, sourceOffset); } outStream.close(); diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/UniformSizeInputFormat.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/UniformSizeInputFormat.java index 3e86d0931bc..d1c18ea8d16 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/UniformSizeInputFormat.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/UniformSizeInputFormat.java @@ -99,7 +99,8 @@ public class UniformSizeInputFormat while (reader.next(srcRelPath, srcFileStatus)) { // If adding the current file would cause the bytes per map to exceed // limit. Add the current file to new split - if (currentSplitSize + srcFileStatus.getLen() > nBytesPerSplit && lastPosition != 0) { + if (currentSplitSize + srcFileStatus.getChunkLength() > nBytesPerSplit + && lastPosition != 0) { FileSplit split = new FileSplit(listingFilePath, lastSplitStart, lastPosition - lastSplitStart, null); if (LOG.isDebugEnabled()) { @@ -109,7 +110,7 @@ public class UniformSizeInputFormat lastSplitStart = lastPosition; currentSplitSize = 0; } - currentSplitSize += srcFileStatus.getLen(); + currentSplitSize += srcFileStatus.getChunkLength(); lastPosition = reader.getPosition(); } if (lastPosition > lastSplitStart) { diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java index 76bc4c56268..e315b848de4 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java @@ -19,9 +19,11 @@ package org.apache.hadoop.tools.util; import com.google.common.collect.Maps; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -30,6 +32,7 @@ import org.apache.hadoop.fs.XAttr; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclUtil; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.InputFormat; @@ -44,6 +47,7 @@ import org.apache.hadoop.util.StringUtils; import java.io.IOException; import java.text.DecimalFormat; import java.util.EnumSet; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -116,7 +120,7 @@ public class DistCpUtils { * @return Class implementing the strategy specified in options. */ public static Class getStrategy(Configuration conf, - DistCpOptions options) { + DistCpOptions options) { String confLabel = "distcp." + StringUtils.toLowerCase(options.getCopyStrategy()) + ".strategy" + ".impl"; @@ -297,6 +301,86 @@ public class DistCpUtils { return fileSystem.getXAttrs(path); } + /** + * Converts FileStatus to a list of CopyListingFileStatus. + * The resulted list contains either one CopyListingFileStatus per chunk of + * file-blocks (if file-size exceeds blockSize * blocksPerChunk, and there + * are more blocks in the file than blocksperChunk), or a single + * CopyListingFileStatus for the entire file (if file-size is too small to + * split). + * If preserving ACLs, populates the CopyListingFileStatus with the ACLs. + * If preserving XAttrs, populates the CopyListingFileStatus with the XAttrs. + * + * @param fileSystem FileSystem containing the file + * @param fileStatus FileStatus of file + * @param preserveAcls boolean true if preserving ACLs + * @param preserveXAttrs boolean true if preserving XAttrs + * @param preserveRawXAttrs boolean true if preserving raw.* XAttrs + * @param blocksPerChunk size of chunks when copying chunks in parallel + * @return list of CopyListingFileStatus + * @throws IOException if there is an I/O error + */ + public static LinkedList toCopyListingFileStatus( + FileSystem fileSystem, FileStatus fileStatus, boolean preserveAcls, + boolean preserveXAttrs, boolean preserveRawXAttrs, int blocksPerChunk) + throws IOException { + LinkedList copyListingFileStatus = + new LinkedList(); + + final CopyListingFileStatus clfs = toCopyListingFileStatusHelper( + fileSystem, fileStatus, preserveAcls, + preserveXAttrs, preserveRawXAttrs, + 0, fileStatus.getLen()); + final long blockSize = fileStatus.getBlockSize(); + if (LOG.isDebugEnabled()) { + LOG.debug("toCopyListing: " + fileStatus + " chunkSize: " + + blocksPerChunk + " isDFS: " + + (fileSystem instanceof DistributedFileSystem)); + } + if ((blocksPerChunk > 0) && + !fileStatus.isDirectory() && + (fileStatus.getLen() > blockSize * blocksPerChunk)) { + // split only when the file size is larger than the intended chunk size + final BlockLocation[] blockLocations; + blockLocations = fileSystem.getFileBlockLocations(fileStatus, 0, + fileStatus.getLen()); + + int numBlocks = blockLocations.length; + long curPos = 0; + if (numBlocks <= blocksPerChunk) { + if (LOG.isDebugEnabled()) { + LOG.debug(" add file " + clfs); + } + copyListingFileStatus.add(clfs); + } else { + int i = 0; + while (i < numBlocks) { + long curLength = 0; + for (int j = 0; j < blocksPerChunk && i < numBlocks; ++j, ++i) { + curLength += blockLocations[i].getLength(); + } + if (curLength > 0) { + CopyListingFileStatus clfs1 = new CopyListingFileStatus(clfs); + clfs1.setChunkOffset(curPos); + clfs1.setChunkLength(curLength); + if (LOG.isDebugEnabled()) { + LOG.debug(" add file chunk " + clfs1); + } + copyListingFileStatus.add(clfs1); + curPos += curLength; + } + } + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug(" add file/dir " + clfs); + } + copyListingFileStatus.add(clfs); + } + + return copyListingFileStatus; + } + /** * Converts a FileStatus to a CopyListingFileStatus. If preserving ACLs, * populates the CopyListingFileStatus with the ACLs. If preserving XAttrs, @@ -307,13 +391,17 @@ public class DistCpUtils { * @param preserveAcls boolean true if preserving ACLs * @param preserveXAttrs boolean true if preserving XAttrs * @param preserveRawXAttrs boolean true if preserving raw.* XAttrs + * @param chunkOffset chunk offset in bytes + * @param chunkLength chunk length in bytes + * @return CopyListingFileStatus * @throws IOException if there is an I/O error */ - public static CopyListingFileStatus toCopyListingFileStatus( + public static CopyListingFileStatus toCopyListingFileStatusHelper( FileSystem fileSystem, FileStatus fileStatus, boolean preserveAcls, - boolean preserveXAttrs, boolean preserveRawXAttrs) throws IOException { + boolean preserveXAttrs, boolean preserveRawXAttrs, + long chunkOffset, long chunkLength) throws IOException { CopyListingFileStatus copyListingFileStatus = - new CopyListingFileStatus(fileStatus); + new CopyListingFileStatus(fileStatus, chunkOffset, chunkLength); if (preserveAcls) { FsPermission perm = fileStatus.getPermission(); if (perm.getAclBit()) { @@ -470,4 +558,19 @@ public class DistCpUtils { return (sourceChecksum == null || targetChecksum == null || sourceChecksum.equals(targetChecksum)); } + + /* + * Return the Path for a given chunk. + * Used when splitting large file into chunks to copy in parallel. + * @param targetFile path to target file + * @param srcFileStatus source file status in copy listing + * @return path to the chunk specified by the parameters to store + * in target cluster temporarily + */ + public static Path getSplitChunkPath(Path targetFile, + CopyListingFileStatus srcFileStatus) { + return new Path(targetFile.toString() + + ".____distcpSplit____" + srcFileStatus.getChunkOffset() + + "." + srcFileStatus.getChunkLength()); + } } diff --git a/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm b/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm index 41a6e94aeee..a77deb2ffee 100644 --- a/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm +++ b/hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm @@ -237,6 +237,7 @@ Flag | Description | Notes `-rdiff ` | Use snapshot diff report between given two snapshots to identify what has been changed on the target since the snapshot `` was created on the target, and apply the diff reversely to the target, and copy modified files from the source's ``, to make the target the same as ``. | This option is valid only with `-update` option and the following conditions should be satisfied.
    1. Both the source and the target FileSystem must be DistributedFileSystem. The source and the target can be two different clusters/paths, or they can be exactly the same cluster/path. In the latter case, modified files are copied from target's `` to target's current state).
    2. Two snapshots `` and `` have been created on the target FS, and `` is older than ``. No change has been made on target since `` was created on the target.
    3. The source has the same snapshot ``, which has the same content as the `` on the target. All the files/directories in the target's `` are the same with source's ``.
    | `-numListstatusThreads` | Number of threads to use for building file listing | At most 40 threads. `-skipcrccheck` | Whether to skip CRC checks between source and target paths. | +`-blocksperchunk ` | Number of blocks per chunk. When specified, split files into chunks to copy in parallel | If set to a positive value, files with more blocks than this value will be split into chunks of `` blocks to be transferred in parallel, and reassembled on the destination. By default, `` is 0 and the files will be transmitted in their entirety without splitting. This switch is only applicable when the source file system implements getBlockLocations method and the target file system implements concat method. | Architecture of DistCp ---------------------- diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java index e3018a07730..b2266b3344d 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java @@ -23,17 +23,27 @@ import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.OutputStream; +import java.io.PrintStream; import java.net.URI; import java.util.ArrayList; import java.util.List; +import java.util.Random; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FsShell; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.util.ToolRunner; import org.junit.AfterClass; import org.junit.Assert; @@ -47,11 +57,15 @@ import org.junit.rules.Timeout; */ public class TestDistCpSystem { + private static final Log LOG = + LogFactory.getLog(TestDistCpSystem.class); + @Rule public Timeout globalTimeout = new Timeout(30000); private static final String SRCDAT = "srcdat"; private static final String DSTDAT = "dstdat"; + private static final long BLOCK_SIZE = 1024; private static MiniDFSCluster cluster; private static Configuration conf; @@ -63,27 +77,76 @@ public class TestDistCpSystem { this.path = path; this.isDir = isDir; } - String getPath() { return path; } - boolean isDirectory() { return isDir; } + + String getPath() { + return path; + } + + boolean isDirectory() { + return isDir; + } + } + + @BeforeClass + public static void beforeClass() throws IOException { + conf = new Configuration(); + conf.setLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, BLOCK_SIZE); + conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); + cluster.waitActive(); + } + + @AfterClass + public static void afterClass() throws IOException { + if (cluster != null) { + cluster.shutdown(); + } + } + + static String execCmd(FsShell shell, String... args) throws Exception { + ByteArrayOutputStream baout = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(baout, true); + PrintStream old = System.out; + System.setOut(out); + shell.run(args); + out.close(); + System.setOut(old); + return baout.toString(); } - private void createFiles(FileSystem fs, String topdir, - FileEntry[] entries) throws IOException { + private void createFiles(DistributedFileSystem fs, String topdir, + FileEntry[] entries, long chunkSize) throws IOException { + long seed = System.currentTimeMillis(); + Random rand = new Random(seed); + short replicationFactor = 2; for (FileEntry entry : entries) { - Path newpath = new Path(topdir + "/" + entry.getPath()); + Path newPath = new Path(topdir + "/" + entry.getPath()); if (entry.isDirectory()) { - fs.mkdirs(newpath); + fs.mkdirs(newPath); } else { - OutputStream out = fs.create(newpath); - try { - out.write((topdir + "/" + entry).getBytes()); - out.write("\n".getBytes()); - } finally { - out.close(); + long fileSize = BLOCK_SIZE *100; + int bufSize = 128; + if (chunkSize == -1) { + DFSTestUtil.createFile(fs, newPath, bufSize, + fileSize, BLOCK_SIZE, replicationFactor, seed); + } else { + // Create a variable length block file, by creating + // one block of half block size at the chunk boundary + long seg1 = chunkSize * BLOCK_SIZE - BLOCK_SIZE / 2; + long seg2 = fileSize - seg1; + DFSTestUtil.createFile(fs, newPath, bufSize, + seg1, BLOCK_SIZE, replicationFactor, seed); + DFSTestUtil.appendFileNewBlock(fs, newPath, (int)seg2); } } + seed = System.currentTimeMillis() + rand.nextLong(); } } + + private void createFiles(DistributedFileSystem fs, String topdir, + FileEntry[] entries) throws IOException { + createFiles(fs, topdir, entries, -1); + } private static FileStatus[] getFileStatus(FileSystem fs, String topdir, FileEntry[] files) throws IOException { @@ -104,18 +167,19 @@ public class TestDistCpSystem { } private void testPreserveUserHelper(String testRoot, - FileEntry[] srcEntries, - FileEntry[] dstEntries, - boolean createSrcDir, - boolean createTgtDir, - boolean update) throws Exception { + FileEntry[] srcEntries, + FileEntry[] dstEntries, + boolean createSrcDir, + boolean createTgtDir, + boolean update) throws Exception { final String testSrcRel = SRCDAT; final String testSrc = testRoot + "/" + testSrcRel; final String testDstRel = DSTDAT; final String testDst = testRoot + "/" + testDstRel; String nnUri = FileSystem.getDefaultUri(conf).toString(); - FileSystem fs = FileSystem.get(URI.create(nnUri), conf); + DistributedFileSystem fs = (DistributedFileSystem) + FileSystem.get(URI.create(nnUri), conf); fs.mkdirs(new Path(testRoot)); if (createSrcDir) { fs.mkdirs(new Path(testSrc)); @@ -129,8 +193,8 @@ public class TestDistCpSystem { for(int i = 0; i < srcEntries.length; i++) { fs.setOwner(srcstats[i].getPath(), "u" + i, null); } - String[] args = update? new String[]{"-pu", "-update", nnUri+testSrc, - nnUri+testDst} : new String[]{"-pu", nnUri+testSrc, nnUri+testDst}; + String[] args = update? new String[]{"-pub", "-update", nnUri+testSrc, + nnUri+testDst} : new String[]{"-pub", nnUri+testSrc, nnUri+testDst}; ToolRunner.run(conf, new DistCp(), args); @@ -145,20 +209,263 @@ public class TestDistCpSystem { deldir(fs, testRoot); } - @BeforeClass - public static void beforeClass() throws IOException { - conf = new Configuration(); - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); - cluster.waitActive(); + private void compareFiles(FileSystem fs, FileStatus srcStat, + FileStatus dstStat) throws Exception { + LOG.info("Comparing " + srcStat + " and " + dstStat); + assertEquals(srcStat.isDirectory(), dstStat.isDirectory()); + assertEquals(srcStat.getReplication(), dstStat.getReplication()); + assertEquals("File POSIX permission should match", + srcStat.getPermission(), dstStat.getPermission()); + assertEquals("File user ownership should match", + srcStat.getOwner(), dstStat.getOwner()); + assertEquals("File group ownership should match", + srcStat.getGroup(), dstStat.getGroup()); + // TODO; check ACL attributes + + if (srcStat.isDirectory()) { + return; + } + + assertEquals("File length should match (" + srcStat.getPath() + ")", + srcStat.getLen(), dstStat.getLen()); + + FSDataInputStream srcIn = fs.open(srcStat.getPath()); + FSDataInputStream dstIn = fs.open(dstStat.getPath()); + try { + byte[] readSrc = new byte[(int) + HdfsClientConfigKeys.DFS_BLOCK_SIZE_DEFAULT]; + byte[] readDst = new byte[(int) + HdfsClientConfigKeys.DFS_BLOCK_SIZE_DEFAULT]; + + int srcBytesRead = 0, tgtBytesRead = 0; + int srcIdx = 0, tgtIdx = 0; + long totalComparedBytes = 0; + while (true) { + if (srcBytesRead == 0) { + srcBytesRead = srcIn.read(readSrc); + srcIdx = 0; + } + if (tgtBytesRead == 0) { + tgtBytesRead = dstIn.read(readDst); + tgtIdx = 0; + } + if (srcBytesRead == 0 || tgtBytesRead == 0) { + LOG.info("______ compared src and dst files for " + + totalComparedBytes + " bytes, content match."); + if (srcBytesRead != tgtBytesRead) { + Assert.fail("Read mismatching size, compared " + + totalComparedBytes + " bytes between src and dst file " + + srcStat + " and " + dstStat); + } + if (totalComparedBytes != srcStat.getLen()) { + Assert.fail("Only read/compared " + totalComparedBytes + + " bytes between src and dst file " + srcStat + + " and " + dstStat); + } else { + // success + break; + } + } + for (; srcIdx < srcBytesRead && tgtIdx < tgtBytesRead; + ++srcIdx, ++tgtIdx) { + if (readSrc[srcIdx] != readDst[tgtIdx]) { + Assert.fail("src and dst file does not match at " + + totalComparedBytes + " between " + + srcStat + " and " + dstStat); + } + ++totalComparedBytes; + } + LOG.info("______ compared src and dst files for " + + totalComparedBytes + " bytes, content match. FileLength: " + + srcStat.getLen()); + if (totalComparedBytes == srcStat.getLen()) { + LOG.info("______ Final:" + srcIdx + " " + + srcBytesRead + " " + tgtIdx + " " + tgtBytesRead); + break; + } + if (srcIdx == srcBytesRead) { + srcBytesRead = 0; + } + if (tgtIdx == tgtBytesRead) { + tgtBytesRead = 0; + } + } + } finally { + if (srcIn != null) { + srcIn.close(); + } + if (dstIn != null) { + dstIn.close(); + } + } } - @AfterClass - public static void afterClass() throws IOException { - if (cluster != null) { - cluster.shutdown(); + // WC: needed because the current distcp does not create target dirs + private void createDestDir(FileSystem fs, String testDst, + FileStatus[] srcStats, FileEntry[] srcFiles) throws IOException { + fs.mkdirs(new Path(testDst)); + + for (int i=0; i=0; --i) { + if (!srcFiles[i].isDirectory()) { + LOG.info("Modifying " + srcStats[i].getPath()); + DFSTestUtil.appendFileNewBlock(fs, srcStats[i].getPath(), + (int)BLOCK_SIZE * 3); + break; + } + } + // get file status after modifying file + srcStats = getFileStatus(fs, testRoot, srcFiles); + + args = new String[] {"-pugp", "-update", "-blocksperchunk", + String.valueOf(chunkSize), + nnUri + testSrc, nnUri + testDst + "/" + testSrcRel}; + + copyAndVerify(fs, srcFiles, srcStats, testDst, args); + + deldir(fs, testRoot); + } + + @Test + public void testRecursiveChunkCopy() throws Exception { + FileEntry[] srcFiles = { + new FileEntry(SRCDAT, true), + new FileEntry(SRCDAT + "/file0", false), + new FileEntry(SRCDAT + "/dir1", true), + new FileEntry(SRCDAT + "/dir2", true), + new FileEntry(SRCDAT + "/dir1/file1", false) + }; + chunkCopy(srcFiles); + } + + @Test + public void testChunkCopyOneFile() throws Exception { + FileEntry[] srcFiles = { + new FileEntry(SRCDAT, true), + new FileEntry(SRCDAT + "/file0", false) + }; + chunkCopy(srcFiles); + } + + @Test + public void testDistcpLargeFile() throws Exception { + FileEntry[] srcfiles = { + new FileEntry(SRCDAT, true), + new FileEntry(SRCDAT + "/file", false) + }; + + final String testRoot = "/testdir"; + final String testSrcRel = SRCDAT; + final String testSrc = testRoot + "/" + testSrcRel; + final String testDstRel = DSTDAT; + final String testDst = testRoot + "/" + testDstRel; + + String nnUri = FileSystem.getDefaultUri(conf).toString(); + DistributedFileSystem fs = + (DistributedFileSystem) FileSystem.get(URI.create(nnUri), conf); + fs.mkdirs(new Path(testRoot)); + fs.mkdirs(new Path(testSrc)); + fs.mkdirs(new Path(testDst)); + long chunkSize = 6; + createFiles(fs, testRoot, srcfiles, chunkSize); + + String srcFileName = testRoot + Path.SEPARATOR + srcfiles[1].getPath(); + Path srcfile = new Path(srcFileName); + + if(!cluster.getFileSystem().exists(srcfile)){ + throw new Exception("src not exist"); + } + + final long srcLen = fs.getFileStatus(srcfile).getLen(); + + FileStatus[] srcstats = getFileStatus(fs, testRoot, srcfiles); + for (int i = 0; i < srcfiles.length; i++) { + fs.setOwner(srcstats[i].getPath(), "u" + i, null); + } + String[] args = new String[] { + "-blocksperchunk", + String.valueOf(chunkSize), + nnUri + testSrc, + nnUri + testDst + }; + + LOG.info("_____ running distcp: " + args[0] + " " + args[1]); + ToolRunner.run(conf, new DistCp(), args); + + String realTgtPath = testDst; + FileStatus[] dststat = getFileStatus(fs, realTgtPath, srcfiles); + assertEquals("File length should match", srcLen, + dststat[dststat.length - 1].getLen()); + + this.compareFiles(fs, srcstats[srcstats.length-1], + dststat[dststat.length-1]); + deldir(fs, testRoot); + } + @Test public void testPreserveUseNonEmptyDir() throws Exception { String testRoot = "/testdir." + getMethodName(); @@ -180,7 +487,6 @@ public class TestDistCpSystem { testPreserveUserHelper(testRoot, srcfiles, dstfiles, false, false, false); } - @Test public void testPreserveUserEmptyDir() throws Exception { String testRoot = "/testdir." + getMethodName(); diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java index efe46272e38..f94ba97ec34 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java @@ -394,7 +394,7 @@ public class TestOptionsParser { + "copyStrategy='uniformsize', preserveStatus=[], " + "preserveRawXattrs=false, atomicWorkPath=null, logPath=null, " + "sourceFileListing=abc, sourcePaths=null, targetPath=xyz, " - + "targetPathExists=true, filtersFile='null'}"; + + "targetPathExists=true, filtersFile='null', blocksPerChunk=0}"; String optionString = option.toString(); Assert.assertEquals(val, optionString); Assert.assertNotSame(DistCpOptionSwitch.ATOMIC_COMMIT.toString(), diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java index 2e9a350b1ca..2452d6fee3d 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java @@ -81,6 +81,10 @@ public class TestCopyCommitter { @Before public void createMetaFolder() { config.set(DistCpConstants.CONF_LABEL_META_FOLDER, "/meta"); + // Unset listing file path since the config is shared by + // multiple tests, and some test doesn't set it, such as + // testNoCommitAction, but the distcp code will check it. + config.set(DistCpConstants.CONF_LABEL_LISTING_FILE_PATH, ""); Path meta = new Path("/meta"); try { cluster.getFileSystem().mkdirs(meta); @@ -326,7 +330,6 @@ public class TestCopyCommitter { committer.commitJob(jobContext); Assert.assertFalse(fs.exists(new Path(workPath))); Assert.assertTrue(fs.exists(new Path(finalPath))); - } catch (IOException e) { LOG.error("Exception encountered while testing for preserve status", e); Assert.fail("Atomic commit failure"); From 28cdc5a8dc37ade1f45bda3aede589ee8593945e Mon Sep 17 00:00:00 2001 From: Hanisha Koneru Date: Thu, 30 Mar 2017 22:41:26 -0700 Subject: [PATCH 174/188] HDFS-11551. Handle SlowDiskReport from DataNode at the NameNode. Contributed by Hanisha Koneru. --- .../hdfs/server/protocol/SlowDiskReports.java | 28 +- .../blockmanagement/DatanodeManager.java | 34 +- .../blockmanagement/SlowDiskTracker.java | 291 ++++++++++++ .../datanode/metrics/DataNodeDiskMetrics.java | 35 +- .../blockmanagement/TestSlowDiskTracker.java | 448 ++++++++++++++++++ 5 files changed, 812 insertions(+), 24 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowDiskTracker.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSlowDiskTracker.java diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/SlowDiskReports.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/SlowDiskReports.java index ef4d09e884d..8095c2a690a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/SlowDiskReports.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/SlowDiskReports.java @@ -48,7 +48,7 @@ public final class SlowDiskReports { private final Map> slowDisks; /** - * An object representing a SlowPeerReports with no entries. Should + * An object representing a SlowDiskReports with no entries. Should * be used instead of null or creating new objects when there are * no slow peers to report. */ @@ -119,8 +119,28 @@ public final class SlowDiskReports { * Lists the types of operations on which disk latencies are measured. */ public enum DiskOp { - METADATA, - READ, - WRITE + METADATA("MetadataOp"), + READ("ReadIO"), + WRITE("WriteIO"); + + private final String value; + + DiskOp(final String v) { + this.value = v; + } + + @Override + public String toString() { + return value; + } + + public static DiskOp fromValue(final String value) { + for (DiskOp as : DiskOp.values()) { + if (as.value.equals(value)) { + return as; + } + } + return null; + } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java index e22b7afa2a3..18135a8a549 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.CachedBlocksList; +import org.apache.hadoop.hdfs.server.common.Util; import org.apache.hadoop.hdfs.server.namenode.CachedBlock; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.Namesystem; @@ -180,9 +181,15 @@ public class DatanodeManager { * True if we should process latency metrics from downstream peers. */ private final boolean dataNodePeerStatsEnabled; + /** + * True if we should process latency metrics from individual DN disks. + */ + private final boolean dataNodeDiskStatsEnabled; @Nullable private final SlowPeerTracker slowPeerTracker; + @Nullable + private final SlowDiskTracker slowDiskTracker; /** * The minimum time between resending caching directives to Datanodes, @@ -208,9 +215,16 @@ public class DatanodeManager { this.dataNodePeerStatsEnabled = conf.getBoolean( DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_KEY, DFSConfigKeys.DFS_DATANODE_PEER_STATS_ENABLED_DEFAULT); + this.dataNodeDiskStatsEnabled = Util.isDiskStatsEnabled(conf.getDouble( + DFSConfigKeys.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY, + DFSConfigKeys.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_DEFAULT)); + final Timer timer = new Timer(); this.slowPeerTracker = dataNodePeerStatsEnabled ? - new SlowPeerTracker(conf, new Timer()) : null; + new SlowPeerTracker(conf, timer) : null; + + this.slowDiskTracker = dataNodeDiskStatsEnabled ? + new SlowDiskTracker(conf, timer) : null; this.defaultXferPort = NetUtils.createSocketAddr( conf.getTrimmed(DFSConfigKeys.DFS_DATANODE_ADDRESS_KEY, @@ -1664,6 +1678,16 @@ public class DatanodeManager { } } + if (slowDiskTracker != null) { + if (!slowDisks.getSlowDisks().isEmpty()) { + if (LOG.isDebugEnabled()) { + LOG.debug("DataNode " + nodeReg + " reported slow disks: " + + slowDisks.getSlowDisks()); + } + slowDiskTracker.addSlowDiskReport(nodeReg.getIpcAddr(false), slowDisks); + } + } + if (!cmds.isEmpty()) { return cmds.toArray(new DatanodeCommand[cmds.size()]); } @@ -1875,5 +1899,13 @@ public class DatanodeManager { public String getSlowPeersReport() { return slowPeerTracker != null ? slowPeerTracker.getJson() : null; } + + /** + * Use only for testing. + */ + @VisibleForTesting + public SlowDiskTracker getSlowDiskTracker() { + return slowDiskTracker; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowDiskTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowDiskTracker.java new file mode 100644 index 00000000000..25920a2e07f --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowDiskTracker.java @@ -0,0 +1,291 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.server.blockmanagement; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.primitives.Doubles; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports; +import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports.DiskOp; +import org.apache.hadoop.util.Timer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Map; +import java.util.PriorityQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * This class aggregates information from {@link SlowDiskReports} received via + * heartbeats. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class SlowDiskTracker { + public static final Logger LOG = + LoggerFactory.getLogger(SlowPeerTracker.class); + + /** + * Time duration after which a report is considered stale. This is + * set to DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY * 3 i.e. + * maintained for at least two successive reports. + */ + private long reportValidityMs; + + /** + * Timer object for querying the current time. Separated out for + * unit testing. + */ + private final Timer timer; + + /** + * Number of disks to include in JSON report per operation. We will return + * disks with the highest latency. + */ + private static final int MAX_DISKS_TO_REPORT = 5; + private static final String DATANODE_DISK_SEPARATOR = ":"; + private final long reportGenerationIntervalMs; + + private volatile long lastUpdateTime; + private AtomicBoolean isUpdateInProgress = new AtomicBoolean(false); + + /** + * Information about disks that have been reported as being slow. + * It is map of (Slow Disk ID) -> (DiskLatency). The DiskLatency contains + * the disk ID, the latencies reported and the timestamp when the report + * was received. + */ + private final Map diskIDLatencyMap; + + /** + * Map of slow disk -> diskOperations it has been reported slow in. + */ + private volatile ArrayList slowDisksReport = + Lists.newArrayList(); + private volatile ArrayList oldSlowDisksCheck; + + public SlowDiskTracker(Configuration conf, Timer timer) { + this.timer = timer; + this.lastUpdateTime = timer.monotonicNow(); + this.diskIDLatencyMap = new ConcurrentHashMap<>(); + this.reportGenerationIntervalMs = conf.getTimeDuration( + DFSConfigKeys.DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY, + DFSConfigKeys.DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_DEFAULT, + TimeUnit.MILLISECONDS); + this.reportValidityMs = reportGenerationIntervalMs * 3; + } + + @VisibleForTesting + public static String getSlowDiskIDForReport(String datanodeID, + String slowDisk) { + return datanodeID + DATANODE_DISK_SEPARATOR + slowDisk; + } + + public void addSlowDiskReport(String dataNodeID, + SlowDiskReports dnSlowDiskReport) { + Map> slowDisks = + dnSlowDiskReport.getSlowDisks(); + + long now = timer.monotonicNow(); + + for (Map.Entry> slowDiskEntry : + slowDisks.entrySet()) { + + String diskID = getSlowDiskIDForReport(dataNodeID, + slowDiskEntry.getKey()); + + Map latencies = slowDiskEntry.getValue(); + + DiskLatency diskLatency = new DiskLatency(diskID, latencies, now); + diskIDLatencyMap.put(diskID, diskLatency); + } + + checkAndUpdateReportIfNecessary(); + } + + private void checkAndUpdateReportIfNecessary() { + // Check if it is time for update + long now = timer.monotonicNow(); + if (now - lastUpdateTime > reportGenerationIntervalMs) { + updateSlowDiskReportAsync(now); + } + } + + @VisibleForTesting + public void updateSlowDiskReportAsync(long now) { + if (isUpdateInProgress.compareAndSet(false, true)) { + lastUpdateTime = now; + new Thread(new Runnable() { + @Override + public void run() { + slowDisksReport = getSlowDisks(diskIDLatencyMap, + MAX_DISKS_TO_REPORT, now); + + cleanUpOldReports(now); + + isUpdateInProgress.set(false); + } + }).start(); + } + } + + /** + * This structure is a thin wrapper over disk latencies. + */ + public static class DiskLatency { + @JsonProperty("SlowDiskID") + final private String slowDiskID; + @JsonProperty("Latencies") + final private Map latencyMap; + @JsonIgnore + private long timestamp; + + /** + * Constructor needed by Jackson for Object mapping. + */ + public DiskLatency( + @JsonProperty("SlowDiskID") String slowDiskID, + @JsonProperty("Latencies") Map latencyMap) { + this.slowDiskID = slowDiskID; + this.latencyMap = latencyMap; + } + + public DiskLatency(String slowDiskID, Map latencyMap, + long timestamp) { + this.slowDiskID = slowDiskID; + this.latencyMap = latencyMap; + this.timestamp = timestamp; + } + + String getSlowDiskID() { + return this.slowDiskID; + } + + double getMaxLatency() { + double maxLatency = 0; + for (double latency : latencyMap.values()) { + if (latency > maxLatency) { + maxLatency = latency; + } + } + return maxLatency; + } + + Double getLatency(DiskOp op) { + return this.latencyMap.get(op); + } + } + + /** + * Retrieve a list of stop low disks i.e disks with the highest max latencies. + * @param numDisks number of disks to return. This is to limit the size of + * the generated JSON. + */ + private ArrayList getSlowDisks( + Map reports, int numDisks, long now) { + if (reports.isEmpty()) { + return new ArrayList(ImmutableList.of()); + } + + final PriorityQueue topNReports = new PriorityQueue<>( + reports.size(), + new Comparator() { + @Override + public int compare(DiskLatency o1, DiskLatency o2) { + return Doubles.compare( + o1.getMaxLatency(), o2.getMaxLatency()); + } + }); + + ArrayList oldSlowDiskIDs = Lists.newArrayList(); + + for (Map.Entry entry : reports.entrySet()) { + DiskLatency diskLatency = entry.getValue(); + if (now - diskLatency.timestamp < reportValidityMs) { + if (topNReports.size() < numDisks) { + topNReports.add(diskLatency); + } else if (topNReports.peek().getMaxLatency() < + diskLatency.getMaxLatency()) { + topNReports.poll(); + topNReports.add(diskLatency); + } + } else { + oldSlowDiskIDs.add(diskLatency); + } + } + + oldSlowDisksCheck = oldSlowDiskIDs; + + return Lists.newArrayList(topNReports); + } + + /** + * Retrieve all valid reports as a JSON string. + * @return serialized representation of valid reports. null if + * serialization failed. + */ + public String getSlowDiskReportAsJsonString() { + ObjectMapper objectMapper = new ObjectMapper(); + try { + return objectMapper.writeValueAsString(slowDisksReport); + } catch (JsonProcessingException e) { + // Failed to serialize. Don't log the exception call stack. + LOG.debug("Failed to serialize statistics" + e); + return null; + } + } + + private void cleanUpOldReports(long now) { + if (oldSlowDisksCheck != null) { + for (DiskLatency oldDiskLatency : oldSlowDisksCheck) { + diskIDLatencyMap.remove(oldDiskLatency.getSlowDiskID(), oldDiskLatency); + } + } + // Replace oldSlowDiskIDsCheck with an empty ArrayList + oldSlowDisksCheck = null; + } + + @VisibleForTesting + ArrayList getSlowDisksReport() { + return this.slowDisksReport; + } + + @VisibleForTesting + long getReportValidityMs() { + return reportValidityMs; + } + + @VisibleForTesting + void setReportValidityMs(long reportValidityMs) { + this.reportValidityMs = reportValidityMs; + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java index 2602b012a64..f2954e824dc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java @@ -20,7 +20,6 @@ package org.apache.hadoop.hdfs.server.datanode.metrics; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; -import com.google.common.collect.Sets; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.hdfs.server.datanode.DataNode; @@ -33,9 +32,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.HashMap; import java.util.Iterator; import java.util.Map; -import java.util.Set; /** * This class detects and maintains DataNode disk outliers and their @@ -122,43 +121,41 @@ public class DataNodeDiskMetrics { private void detectAndUpdateDiskOutliers(Map metadataOpStats, Map readIoStats, Map writeIoStats) { - Set diskOutliersSet = Sets.newHashSet(); + Map> diskStats = Maps.newHashMap(); // Get MetadataOp Outliers Map metadataOpOutliers = slowDiskDetector .getOutliers(metadataOpStats); - if (!metadataOpOutliers.isEmpty()) { - diskOutliersSet.addAll(metadataOpOutliers.keySet()); + for (Map.Entry entry : metadataOpOutliers.entrySet()) { + addDiskStat(diskStats, entry.getKey(), DiskOp.METADATA, entry.getValue()); } // Get ReadIo Outliers Map readIoOutliers = slowDiskDetector .getOutliers(readIoStats); - if (!readIoOutliers.isEmpty()) { - diskOutliersSet.addAll(readIoOutliers.keySet()); + for (Map.Entry entry : readIoOutliers.entrySet()) { + addDiskStat(diskStats, entry.getKey(), DiskOp.READ, entry.getValue()); } // Get WriteIo Outliers Map writeIoOutliers = slowDiskDetector .getOutliers(writeIoStats); - if (!readIoOutliers.isEmpty()) { - diskOutliersSet.addAll(writeIoOutliers.keySet()); - } - - Map> diskStats = - Maps.newHashMap(); - for (String disk : diskOutliersSet) { - Map diskStat = Maps.newHashMap(); - diskStat.put(DiskOp.METADATA, metadataOpStats.get(disk)); - diskStat.put(DiskOp.READ, readIoStats.get(disk)); - diskStat.put(DiskOp.WRITE, writeIoStats.get(disk)); - diskStats.put(disk, diskStat); + for (Map.Entry entry : writeIoOutliers.entrySet()) { + addDiskStat(diskStats, entry.getKey(), DiskOp.WRITE, entry.getValue()); } diskOutliersStats = diskStats; LOG.debug("Updated disk outliers."); } + private void addDiskStat(Map> diskStats, + String disk, DiskOp diskOp, double latency) { + if (!diskStats.containsKey(disk)) { + diskStats.put(disk, new HashMap<>()); + } + diskStats.get(disk).put(diskOp, latency); + } + public Map> getDiskOutliersStats() { return diskOutliersStats; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSlowDiskTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSlowDiskTracker.java new file mode 100644 index 00000000000..e96b96a25af --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSlowDiskTracker.java @@ -0,0 +1,448 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.server.blockmanagement; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableMap; +import org.apache.hadoop.conf.Configuration; +import static org.apache.hadoop.hdfs.DFSConfigKeys + .DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys + .DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.server.datanode.DataNode; +import org.apache.hadoop.hdfs.server.namenode.NameNode; +import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports; +import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports.DiskOp; +import org.apache.hadoop.hdfs.server.blockmanagement.SlowDiskTracker + .DiskLatency; +import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.util.FakeTimer; + +import com.google.common.base.Supplier; +import com.google.common.collect.Maps; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * Tests for {@link SlowDiskTracker}. + */ +public class TestSlowDiskTracker { + public static final Logger LOG = LoggerFactory.getLogger( + TestSlowDiskTracker.class); + + /** + * Set a timeout for every test case. + */ + @Rule + public Timeout testTimeout = new Timeout(300_000); + + private static Configuration conf; + private SlowDiskTracker tracker; + private FakeTimer timer; + private long reportValidityMs; + private static final long OUTLIERS_REPORT_INTERVAL = 1000; + + static { + conf = new HdfsConfiguration(); + conf.setLong(DFS_HEARTBEAT_INTERVAL_KEY, 1L); + conf.setDouble(DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY, 1.0); + conf.setTimeDuration(DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY, + OUTLIERS_REPORT_INTERVAL, TimeUnit.MILLISECONDS); + } + @Before + public void setup() { + timer = new FakeTimer(); + tracker = new SlowDiskTracker(conf, timer); + reportValidityMs = tracker.getReportValidityMs(); + } + + @Test + public void testDataNodeHeartbeatSlowDiskReport() throws Exception { + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2) + .build(); + try { + DataNode dn1 = cluster.getDataNodes().get(0); + DataNode dn2 = cluster.getDataNodes().get(1); + NameNode nn = cluster.getNameNode(0); + + DatanodeManager datanodeManager = nn.getNamesystem().getBlockManager() + .getDatanodeManager(); + SlowDiskTracker slowDiskTracker = datanodeManager.getSlowDiskTracker(); + slowDiskTracker.setReportValidityMs(OUTLIERS_REPORT_INTERVAL * 100); + + dn1.getDiskMetrics().addSlowDiskForTesting("disk1", ImmutableMap.of( + DiskOp.WRITE, 1.3)); + dn1.getDiskMetrics().addSlowDiskForTesting("disk2", ImmutableMap.of( + DiskOp.READ, 1.6, DiskOp.WRITE, 1.1)); + dn2.getDiskMetrics().addSlowDiskForTesting("disk1", ImmutableMap.of( + DiskOp.METADATA, 0.8)); + dn2.getDiskMetrics().addSlowDiskForTesting("disk2", ImmutableMap.of( + DiskOp.WRITE, 1.3)); + + String dn1ID = dn1.getDatanodeId().getIpcAddr(false); + String dn2ID = dn2.getDatanodeId().getIpcAddr(false); + + // Advance the timer and wait for NN to receive reports from DataNodes. + Thread.sleep(OUTLIERS_REPORT_INTERVAL); + + // Wait for NN to receive reports from all DNs + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + return (slowDiskTracker.getSlowDisksReport().size() == 4); + } + }, 1000, 100000); + + Map slowDisksReport = getSlowDisksReportForTesting( + slowDiskTracker); + + assertThat(slowDisksReport.size(), is(4)); + assertTrue(Math.abs(slowDisksReport.get(dn1ID + ":disk1") + .getLatency(DiskOp.WRITE) - 1.3) < 0.0000001); + assertTrue(Math.abs(slowDisksReport.get(dn1ID + ":disk2") + .getLatency(DiskOp.READ) - 1.6) < 0.0000001); + assertTrue(Math.abs(slowDisksReport.get(dn1ID + ":disk2") + .getLatency(DiskOp.WRITE) - 1.1) < 0.0000001); + assertTrue(Math.abs(slowDisksReport.get(dn2ID + ":disk1") + .getLatency(DiskOp.METADATA) - 0.8) < 0.0000001); + assertTrue(Math.abs(slowDisksReport.get(dn2ID + ":disk2") + .getLatency(DiskOp.WRITE) - 1.3) < 0.0000001); + + // Test the slow disk report JSON string + ArrayList jsonReport = getAndDeserializeJson( + slowDiskTracker.getSlowDiskReportAsJsonString()); + + assertThat(jsonReport.size(), is(4)); + assertTrue(isDiskInReports(jsonReport, dn1ID, "disk1", DiskOp.WRITE, 1.3)); + assertTrue(isDiskInReports(jsonReport, dn1ID, "disk2", DiskOp.READ, 1.6)); + assertTrue(isDiskInReports(jsonReport, dn1ID, "disk2", DiskOp.WRITE, 1.1)); + assertTrue(isDiskInReports(jsonReport, dn2ID, "disk1", DiskOp.METADATA, + 0.8)); + assertTrue(isDiskInReports(jsonReport, dn2ID, "disk2", DiskOp.WRITE, 1.3)); + } finally { + cluster.shutdown(); + } + } + + /** + * Edge case, there are no reports to retrieve. + */ + @Test + public void testEmptyReports() { + tracker.updateSlowDiskReportAsync(timer.monotonicNow()); + assertTrue(getSlowDisksReportForTesting(tracker).isEmpty()); + } + + @Test + public void testReportsAreRetrieved() throws Exception { + addSlowDiskForTesting("dn1", "disk1", + ImmutableMap.of(DiskOp.METADATA, 1.1, DiskOp.READ, 1.8)); + addSlowDiskForTesting("dn1", "disk2", + ImmutableMap.of(DiskOp.READ, 1.3)); + addSlowDiskForTesting("dn2", "disk2", + ImmutableMap.of(DiskOp.READ, 1.1)); + + tracker.updateSlowDiskReportAsync(timer.monotonicNow()); + + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + return !tracker.getSlowDisksReport().isEmpty(); + } + }, 500, 5000); + + Map reports = getSlowDisksReportForTesting(tracker); + + assertThat(reports.size(), is(3)); + assertTrue(Math.abs(reports.get("dn1:disk1") + .getLatency(DiskOp.METADATA) - 1.1) < 0.0000001); + assertTrue(Math.abs(reports.get("dn1:disk1") + .getLatency(DiskOp.READ) - 1.8) < 0.0000001); + assertTrue(Math.abs(reports.get("dn1:disk2") + .getLatency(DiskOp.READ) - 1.3) < 0.0000001); + assertTrue(Math.abs(reports.get("dn2:disk2") + .getLatency(DiskOp.READ) - 1.1) < 0.0000001); + } + + /** + * Test that when all reports are expired, we get back nothing. + */ + @Test + public void testAllReportsAreExpired() throws Exception { + addSlowDiskForTesting("dn1", "disk1", + ImmutableMap.of(DiskOp.METADATA, 1.1, DiskOp.READ, 1.8)); + addSlowDiskForTesting("dn1", "disk2", + ImmutableMap.of(DiskOp.READ, 1.3)); + addSlowDiskForTesting("dn2", "disk2", + ImmutableMap.of(DiskOp.WRITE, 1.1)); + + // No reports should expire after 1ms. + timer.advance(1); + tracker.updateSlowDiskReportAsync(timer.monotonicNow()); + + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + return !tracker.getSlowDisksReport().isEmpty(); + } + }, 500, 5000); + + Map reports = getSlowDisksReportForTesting(tracker); + + assertThat(reports.size(), is(3)); + assertTrue(Math.abs(reports.get("dn1:disk1") + .getLatency(DiskOp.METADATA) - 1.1) < 0.0000001); + assertTrue(Math.abs(reports.get("dn1:disk1") + .getLatency(DiskOp.READ) - 1.8) < 0.0000001); + assertTrue(Math.abs(reports.get("dn1:disk2") + .getLatency(DiskOp.READ) - 1.3) < 0.0000001); + assertTrue(Math.abs(reports.get("dn2:disk2") + .getLatency(DiskOp.WRITE) - 1.1) < 0.0000001); + + // All reports should expire after REPORT_VALIDITY_MS. + timer.advance(reportValidityMs); + tracker.updateSlowDiskReportAsync(timer.monotonicNow()); + + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + return tracker.getSlowDisksReport().isEmpty(); + } + }, 500, 3000); + + reports = getSlowDisksReportForTesting(tracker); + + assertThat(reports.size(), is(0)); + } + + /** + * Test the case when a subset of reports has expired. + * Ensure that we only get back non-expired reports. + */ + @Test + public void testSomeReportsAreExpired() throws Exception { + addSlowDiskForTesting("dn1", "disk1", + ImmutableMap.of(DiskOp.METADATA, 1.1, DiskOp.READ, 1.8)); + addSlowDiskForTesting("dn1", "disk2", + ImmutableMap.of(DiskOp.READ, 1.3)); + timer.advance(reportValidityMs); + addSlowDiskForTesting("dn2", "disk2", + ImmutableMap.of(DiskOp.WRITE, 1.1)); + + tracker.updateSlowDiskReportAsync(timer.monotonicNow()); + + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + return !tracker.getSlowDisksReport().isEmpty(); + } + }, 500, 5000); + + Map reports = getSlowDisksReportForTesting(tracker); + + assertThat(reports.size(), is(1)); + assertTrue(Math.abs(reports.get("dn2:disk2") + .getLatency(DiskOp.WRITE) - 1.1) < 0.0000001); + } + + /** + * Test the case when an expired report is replaced by a valid one. + */ + @Test + public void testReplacement() throws Exception { + addSlowDiskForTesting("dn1", "disk1", + ImmutableMap.of(DiskOp.METADATA, 1.1, DiskOp.READ, 1.8)); + timer.advance(reportValidityMs); + addSlowDiskForTesting("dn1", "disk1", + ImmutableMap.of(DiskOp.READ, 1.4)); + + tracker.updateSlowDiskReportAsync(timer.monotonicNow()); + + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + return !tracker.getSlowDisksReport().isEmpty(); + } + }, 500, 5000); + + Map reports = getSlowDisksReportForTesting(tracker); + + assertThat(reports.size(), is(1)); + assertTrue(reports.get("dn1:disk1").getLatency(DiskOp.METADATA) == null); + assertTrue(Math.abs(reports.get("dn1:disk1") + .getLatency(DiskOp.READ) - 1.4) < 0.0000001); + } + + @Test + public void testGetJson() throws Exception { + addSlowDiskForTesting("dn1", "disk1", + ImmutableMap.of(DiskOp.METADATA, 1.1, DiskOp.READ, 1.8)); + addSlowDiskForTesting("dn1", "disk2", + ImmutableMap.of(DiskOp.READ, 1.3)); + addSlowDiskForTesting("dn2", "disk2", + ImmutableMap.of(DiskOp.WRITE, 1.1)); + addSlowDiskForTesting("dn3", "disk1", + ImmutableMap.of(DiskOp.WRITE, 1.1)); + + tracker.updateSlowDiskReportAsync(timer.monotonicNow()); + + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + return tracker.getSlowDiskReportAsJsonString() != null; + } + }, 500, 5000); + + ArrayList jsonReport = getAndDeserializeJson( + tracker.getSlowDiskReportAsJsonString()); + + // And ensure its contents are what we expect. + assertThat(jsonReport.size(), is(4)); + assertTrue(isDiskInReports(jsonReport, "dn1", "disk1", DiskOp.METADATA, + 1.1)); + assertTrue(isDiskInReports(jsonReport, "dn1", "disk1", DiskOp.READ, 1.8)); + assertTrue(isDiskInReports(jsonReport, "dn1", "disk2", DiskOp.READ, 1.3)); + assertTrue(isDiskInReports(jsonReport, "dn2", "disk2", DiskOp.WRITE, 1.1)); + assertTrue(isDiskInReports(jsonReport, "dn3", "disk1", DiskOp.WRITE, 1.1)); + } + + @Test + public void testGetJsonSizeIsLimited() throws Exception { + addSlowDiskForTesting("dn1", "disk1", + ImmutableMap.of(DiskOp.READ, 1.1)); + addSlowDiskForTesting("dn1", "disk2", + ImmutableMap.of(DiskOp.READ, 1.2)); + addSlowDiskForTesting("dn1", "disk3", + ImmutableMap.of(DiskOp.READ, 1.3)); + addSlowDiskForTesting("dn2", "disk1", + ImmutableMap.of(DiskOp.READ, 1.4)); + addSlowDiskForTesting("dn2", "disk2", + ImmutableMap.of(DiskOp.READ, 1.5)); + addSlowDiskForTesting("dn3", "disk1", + ImmutableMap.of(DiskOp.WRITE, 1.6)); + addSlowDiskForTesting("dn3", "disk2", + ImmutableMap.of(DiskOp.READ, 1.7)); + addSlowDiskForTesting("dn3", "disk3", + ImmutableMap.of(DiskOp.READ, 1.2)); + + tracker.updateSlowDiskReportAsync(timer.monotonicNow()); + + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + return tracker.getSlowDiskReportAsJsonString() != null; + } + }, 500, 5000); + + ArrayList jsonReport = getAndDeserializeJson( + tracker.getSlowDiskReportAsJsonString()); + + // Ensure that only the top 5 highest latencies are in the report. + assertThat(jsonReport.size(), is(5)); + assertTrue(isDiskInReports(jsonReport, "dn3", "disk2", DiskOp.READ, 1.7)); + assertTrue(isDiskInReports(jsonReport, "dn3", "disk1", DiskOp.WRITE, 1.6)); + assertTrue(isDiskInReports(jsonReport, "dn2", "disk2", DiskOp.READ, 1.5)); + assertTrue(isDiskInReports(jsonReport, "dn2", "disk1", DiskOp.READ, 1.4)); + assertTrue(isDiskInReports(jsonReport, "dn1", "disk3", DiskOp.READ, 1.3)); + + // Remaining nodes should be in the list. + assertFalse(isDiskInReports(jsonReport, "dn1", "disk1", DiskOp.READ, 1.1)); + assertFalse(isDiskInReports(jsonReport, "dn1", "disk2", DiskOp.READ, 1.2)); + assertFalse(isDiskInReports(jsonReport, "dn3", "disk3", DiskOp.READ, 1.2)); + } + + @Test + public void testEmptyReport() throws Exception { + addSlowDiskForTesting("dn1", "disk1", + ImmutableMap.of(DiskOp.READ, 1.1)); + timer.advance(reportValidityMs); + + tracker.updateSlowDiskReportAsync(timer.monotonicNow()); + + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + return tracker.getSlowDiskReportAsJsonString() != null; + } + }, 500, 5000); + + ArrayList jsonReport = getAndDeserializeJson( + tracker.getSlowDiskReportAsJsonString()); + + assertTrue(jsonReport.isEmpty()); + } + + private boolean isDiskInReports(ArrayList reports, + String dataNodeID, String disk, DiskOp diskOp, double latency) { + String diskID = SlowDiskTracker.getSlowDiskIDForReport(dataNodeID, disk); + for (DiskLatency diskLatency : reports) { + if (diskLatency.getSlowDiskID().equals(diskID)) { + if (diskLatency.getLatency(diskOp) == null) { + return false; + } + if (Math.abs(diskLatency.getLatency(diskOp) - latency) < 0.0000001) { + return true; + } + } + } + return false; + } + + private ArrayList getAndDeserializeJson( + final String json) throws IOException { + return (new ObjectMapper()).readValue(json, + new TypeReference>() {}); + } + + private void addSlowDiskForTesting(String dnID, String disk, + Map latencies) { + Map> slowDisk = Maps.newHashMap(); + slowDisk.put(disk, latencies); + SlowDiskReports slowDiskReport = SlowDiskReports.create(slowDisk); + tracker.addSlowDiskReport(dnID, slowDiskReport); + } + + Map getSlowDisksReportForTesting( + SlowDiskTracker slowDiskTracker) { + Map slowDisksMap = Maps.newHashMap(); + for (DiskLatency diskLatency : slowDiskTracker.getSlowDisksReport()) { + slowDisksMap.put(diskLatency.getSlowDiskID(), diskLatency); + } + return slowDisksMap; + } +} From 4d1fac5df2508011adfc4c5d683beb00fd5ced45 Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Fri, 31 Mar 2017 10:05:34 -0500 Subject: [PATCH 175/188] YARN-6411. Clean up the overwrite of createDispatcher() in subclass of MockRM. Contributed by Yufei Gu --- .../v2/app/rm/TestRMContainerAllocator.java | 453 ++++++++---------- .../api/impl/TestAMRMClientOnRMRestart.java | 59 +-- .../server/resourcemanager/ACLsTestBase.java | 10 - .../server/resourcemanager/RMHATestBase.java | 20 +- .../ReservationACLsTestBase.java | 5 +- .../TestApplicationCleanup.java | 44 +- .../TestApplicationMasterLauncher.java | 11 +- .../TestApplicationMasterService.java | 19 +- .../TestNodeBlacklistingOnAMFailures.java | 41 +- .../TestReservationSystemWithRMHA.java | 5 +- .../TestAMRMRPCNodeUpdates.java | 18 +- .../resourcetracker/TestNMReconnect.java | 14 +- .../rmcontainer/TestRMContainerImpl.java | 1 - .../capacity/TestApplicationPriority.java | 29 +- .../security/TestClientToAMTokens.java | 23 +- 15 files changed, 277 insertions(+), 475 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java index e6aee6ea997..933bd013ef6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java @@ -179,21 +179,19 @@ public class TestRMContainerAllocator { Configuration conf = new Configuration(); MyResourceManager rm = new MyResourceManager(conf); rm.start(); - DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext() - .getDispatcher(); // Submit the application RMApp app = rm.submitApp(1024); - dispatcher.await(); + rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); amNodeManager.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt() .getAppAttemptId(); rm.sendAMLaunched(appAttemptId); - dispatcher.await(); + rm.drainEvents(); JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0); Job mockJob = mock(Job.class); @@ -207,7 +205,7 @@ public class TestRMContainerAllocator { MockNM nodeManager1 = rm.registerNode("h1:1234", 10240); MockNM nodeManager2 = rm.registerNode("h2:1234", 10240); MockNM nodeManager3 = rm.registerNode("h3:1234", 10240); - dispatcher.await(); + rm.drainEvents(); // create the container request ContainerRequestEvent event1 = createReq(jobId, 1, 1024, @@ -222,7 +220,7 @@ public class TestRMContainerAllocator { // this tells the scheduler about the requests // as nodes are not added, no allocations List assigned = allocator.schedule(); - dispatcher.await(); + rm.drainEvents(); Assert.assertEquals("No of assignments must be 0", 0, assigned.size()); Assert.assertEquals(4, rm.getMyFifoScheduler().lastAsk.size()); @@ -234,7 +232,7 @@ public class TestRMContainerAllocator { // this tells the scheduler about the requests // as nodes are not added, no allocations assigned = allocator.schedule(); - dispatcher.await(); + rm.drainEvents(); Assert.assertEquals("No of assignments must be 0", 0, assigned.size()); Assert.assertEquals(3, rm.getMyFifoScheduler().lastAsk.size()); @@ -242,18 +240,18 @@ public class TestRMContainerAllocator { nodeManager1.nodeHeartbeat(true); // Node heartbeat nodeManager2.nodeHeartbeat(true); // Node heartbeat nodeManager3.nodeHeartbeat(true); // Node heartbeat - dispatcher.await(); + rm.drainEvents(); assigned = allocator.schedule(); - dispatcher.await(); + rm.drainEvents(); Assert.assertEquals(0, rm.getMyFifoScheduler().lastAsk.size()); checkAssignments(new ContainerRequestEvent[] { event1, event2, event3 }, assigned, false); // check that the assigned container requests are cancelled - assigned = allocator.schedule(); - dispatcher.await(); - Assert.assertEquals(5, rm.getMyFifoScheduler().lastAsk.size()); + allocator.schedule(); + rm.drainEvents(); + Assert.assertEquals(5, rm.getMyFifoScheduler().lastAsk.size()); } @Test @@ -269,21 +267,19 @@ public class TestRMContainerAllocator { Configuration conf = new Configuration(); MyResourceManager rm = new MyResourceManager(conf); rm.start(); - DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext() - .getDispatcher(); // Submit the application RMApp app = rm.submitApp(1024); - dispatcher.await(); + rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); amNodeManager.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt() .getAppAttemptId(); rm.sendAMLaunched(appAttemptId); - dispatcher.await(); + rm.drainEvents(); JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0); Job mockJob = mock(Job.class); @@ -297,7 +293,7 @@ public class TestRMContainerAllocator { MockNM nodeManager1 = rm.registerNode("h1:1234", 3072); // can assign 2 maps rm.registerNode("h2:1234", 10240); // wont heartbeat on node local node MockNM nodeManager3 = rm.registerNode("h3:1234", 1536); // assign 1 map - dispatcher.await(); + rm.drainEvents(); // create the container requests for maps ContainerRequestEvent event1 = createReq(jobId, 1, 1024, @@ -313,7 +309,7 @@ public class TestRMContainerAllocator { // this tells the scheduler about the requests // as nodes are not added, no allocations List assigned = allocator.schedule(); - dispatcher.await(); + rm.drainEvents(); Assert.assertEquals("No of assignments must be 0", 0, assigned.size()); // update resources in scheduler @@ -323,10 +319,10 @@ public class TestRMContainerAllocator { // Node heartbeat from node-local next. This allocates 2 node local // containers for task1 and task2. These should be matched with those tasks. nodeManager1.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); assigned = allocator.schedule(); - dispatcher.await(); + rm.drainEvents(); checkAssignments(new ContainerRequestEvent[] { event1, event2, event3 }, assigned, false); // remove the rack-local assignment that should have happened for task3 @@ -350,21 +346,19 @@ public class TestRMContainerAllocator { Configuration conf = new Configuration(); MyResourceManager rm = new MyResourceManager(conf); rm.start(); - DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext() - .getDispatcher(); // Submit the application RMApp app = rm.submitApp(1024); - dispatcher.await(); + rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); amNodeManager.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt() .getAppAttemptId(); rm.sendAMLaunched(appAttemptId); - dispatcher.await(); + rm.drainEvents(); JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0); Job mockJob = mock(Job.class); @@ -378,7 +372,7 @@ public class TestRMContainerAllocator { MockNM nodeManager1 = rm.registerNode("h1:1234", 10240); MockNM nodeManager2 = rm.registerNode("h2:1234", 10240); MockNM nodeManager3 = rm.registerNode("h3:1234", 10240); - dispatcher.await(); + rm.drainEvents(); // create the container request ContainerRequestEvent event1 = createReq(jobId, 1, 1024, @@ -393,17 +387,17 @@ public class TestRMContainerAllocator { // this tells the scheduler about the requests // as nodes are not added, no allocations List assigned = allocator.schedule(); - dispatcher.await(); + rm.drainEvents(); Assert.assertEquals("No of assignments must be 0", 0, assigned.size()); // update resources in scheduler nodeManager1.nodeHeartbeat(true); // Node heartbeat nodeManager2.nodeHeartbeat(true); // Node heartbeat nodeManager3.nodeHeartbeat(true); // Node heartbeat - dispatcher.await(); + rm.drainEvents(); assigned = allocator.schedule(); - dispatcher.await(); + rm.drainEvents(); checkAssignments(new ContainerRequestEvent[] { event1, event2 }, assigned, false); } @@ -416,19 +410,17 @@ public class TestRMContainerAllocator { conf.setFloat(MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART, 0.0f); final MyResourceManager rm = new MyResourceManager(conf); rm.start(); - final DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext() - .getDispatcher(); final RMApp app = rm.submitApp(1024); - dispatcher.await(); + rm.drainEvents(); final String host = "host1"; final MockNM nm = rm.registerNode(String.format("%s:1234", host), 2048); nm.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); final ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt() .getAppAttemptId(); rm.sendAMLaunched(appAttemptId); - dispatcher.await(); + rm.drainEvents(); final JobId jobId = MRBuilderUtils .newJobId(appAttemptId.getApplicationId(), 0); final Job mockJob = mock(Job.class); @@ -438,20 +430,20 @@ public class TestRMContainerAllocator { final MyContainerAllocator allocator = new MyContainerAllocator(rm, conf, appAttemptId, mockJob, SystemClock.getInstance()); // add resources to scheduler - dispatcher.await(); + rm.drainEvents(); // create the container request final String[] locations = new String[] { host }; allocator.sendRequest(createReq(jobId, 0, 1024, locations, false, true)); for (int i = 0; i < 1;) { - dispatcher.await(); + rm.drainEvents(); i += allocator.schedule().size(); nm.nodeHeartbeat(true); } allocator.sendRequest(createReq(jobId, 0, 1024, locations, true, false)); while (allocator.getTaskAttemptKillEvents().size() == 0) { - dispatcher.await(); + rm.drainEvents(); allocator.schedule().size(); nm.nodeHeartbeat(true); } @@ -468,21 +460,19 @@ public class TestRMContainerAllocator { Configuration conf = new Configuration(); MyResourceManager rm = new MyResourceManager(conf); rm.start(); - DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext() - .getDispatcher(); // Submit the application RMApp app = rm.submitApp(1024); - dispatcher.await(); + rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); amNodeManager.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt() .getAppAttemptId(); rm.sendAMLaunched(appAttemptId); - dispatcher.await(); + rm.drainEvents(); JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0); Job mockJob = mock(Job.class); @@ -521,21 +511,19 @@ public class TestRMContainerAllocator { MyResourceManager rm = new MyResourceManager(conf); rm.start(); - DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext() - .getDispatcher(); // Submit the application RMApp app = rm.submitApp(1024); - dispatcher.await(); + rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); amNodeManager.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt() .getAppAttemptId(); rm.sendAMLaunched(appAttemptId); - dispatcher.await(); + rm.drainEvents(); JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0); Job mockJob = mock(Job.class); @@ -584,21 +572,19 @@ public class TestRMContainerAllocator { MyResourceManager rm = new MyResourceManager(conf); rm.start(); rm.getMyFifoScheduler().forceResourceLimit(Resource.newInstance(8192, 8)); - DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext() - .getDispatcher(); // Submit the application RMApp app = rm.submitApp(1024); - dispatcher.await(); + rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); amNodeManager.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt() .getAppAttemptId(); rm.sendAMLaunched(appAttemptId); - dispatcher.await(); + rm.drainEvents(); JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0); Job mockJob = mock(Job.class); @@ -639,18 +625,16 @@ public class TestRMContainerAllocator { conf.setFloat(MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART, 0.0f); final MyResourceManager2 rm = new MyResourceManager2(conf); rm.start(); - final DrainDispatcher dispatcher = (DrainDispatcher)rm.getRMContext() - .getDispatcher(); final RMApp app = rm.submitApp(2048); - dispatcher.await(); + rm.drainEvents(); final String host = "host1"; final MockNM nm = rm.registerNode(String.format("%s:1234", host), 4096); nm.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); final ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt() .getAppAttemptId(); rm.sendAMLaunched(appAttemptId); - dispatcher.await(); + rm.drainEvents(); final JobId jobId = MRBuilderUtils .newJobId(appAttemptId.getApplicationId(), 0); final Job mockJob = mock(Job.class); @@ -666,14 +650,14 @@ public class TestRMContainerAllocator { allocator.scheduleAllReduces(); allocator.makeRemoteRequest(); nm.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); allocator.sendRequest(createReq(jobId, 1, 1024, locations, false, false)); int assignedContainer; for (assignedContainer = 0; assignedContainer < 1;) { assignedContainer += allocator.schedule().size(); nm.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); } // only 1 allocated container should be assigned Assert.assertEquals(assignedContainer, 1); @@ -773,21 +757,19 @@ public class TestRMContainerAllocator { Configuration conf = new Configuration(); MyResourceManager rm = new MyResourceManager(conf); rm.start(); - DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext() - .getDispatcher(); // Submit the application RMApp app = rm.submitApp(1024); - dispatcher.await(); + rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); amNodeManager.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt() .getAppAttemptId(); rm.sendAMLaunched(appAttemptId); - dispatcher.await(); + rm.drainEvents(); JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0); Job mockJob = mock(Job.class); @@ -801,7 +783,7 @@ public class TestRMContainerAllocator { MockNM nodeManager1 = rm.registerNode("h1:1234", 1024); MockNM nodeManager2 = rm.registerNode("h2:1234", 10240); MockNM nodeManager3 = rm.registerNode("h3:1234", 10240); - dispatcher.await(); + rm.drainEvents(); // create the container request // send MAP request @@ -822,17 +804,17 @@ public class TestRMContainerAllocator { // this tells the scheduler about the requests // as nodes are not added, no allocations List assigned = allocator.schedule(); - dispatcher.await(); + rm.drainEvents(); Assert.assertEquals("No of assignments must be 0", 0, assigned.size()); // update resources in scheduler nodeManager1.nodeHeartbeat(true); // Node heartbeat nodeManager2.nodeHeartbeat(true); // Node heartbeat nodeManager3.nodeHeartbeat(true); // Node heartbeat - dispatcher.await(); + rm.drainEvents(); assigned = allocator.schedule(); - dispatcher.await(); + rm.drainEvents(); checkAssignments(new ContainerRequestEvent[] { event1, event3 }, assigned, false); @@ -863,11 +845,6 @@ public class TestRMContainerAllocator { MyResourceManager.setClusterTimeStamp(fakeClusterTimeStamp); } - @Override - protected Dispatcher createDispatcher() { - return new DrainDispatcher(); - } - @Override protected EventHandler createSchedulerEventDispatcher() { // Dispatch inline for test sanity @@ -912,16 +889,16 @@ public class TestRMContainerAllocator { // Submit the application RMApp rmApp = rm.submitApp(1024); - rmDispatcher.await(); + rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 21504); amNodeManager.nodeHeartbeat(true); - rmDispatcher.await(); + rm.drainEvents(); final ApplicationAttemptId appAttemptId = rmApp.getCurrentAppAttempt() .getAppAttemptId(); rm.sendAMLaunched(appAttemptId); - rmDispatcher.await(); + rm.drainEvents(); MRApp mrApp = new MRApp(appAttemptId, ContainerId.newContainerId( appAttemptId, 0), 10, 10, false, this.getClass().getName(), true, 1) { @@ -959,11 +936,11 @@ public class TestRMContainerAllocator { amDispatcher.await(); allocator.schedule(); - rmDispatcher.await(); + rm.drainEvents(); amNodeManager.nodeHeartbeat(true); - rmDispatcher.await(); + rm.drainEvents(); allocator.schedule(); - rmDispatcher.await(); + rm.drainEvents(); // Wait for all map-tasks to be running for (Task t : job.getTasks().values()) { @@ -973,7 +950,7 @@ public class TestRMContainerAllocator { } allocator.schedule(); // Send heartbeat - rmDispatcher.await(); + rm.drainEvents(); Assert.assertEquals(0.05f, job.getProgress(), 0.001f); Assert.assertEquals(0.05f, rmApp.getProgress(), 0.001f); @@ -981,14 +958,14 @@ public class TestRMContainerAllocator { Iterator it = job.getTasks().values().iterator(); finishNextNTasks(rmDispatcher, amNodeManager, mrApp, it, 1); allocator.schedule(); - rmDispatcher.await(); + rm.drainEvents(); Assert.assertEquals(0.095f, job.getProgress(), 0.001f); Assert.assertEquals(0.095f, rmApp.getProgress(), 0.001f); // Finish off 7 more so that map-progress is 80% finishNextNTasks(rmDispatcher, amNodeManager, mrApp, it, 7); allocator.schedule(); - rmDispatcher.await(); + rm.drainEvents(); Assert.assertEquals(0.41f, job.getProgress(), 0.001f); Assert.assertEquals(0.41f, rmApp.getProgress(), 0.001f); @@ -996,11 +973,11 @@ public class TestRMContainerAllocator { finishNextNTasks(rmDispatcher, amNodeManager, mrApp, it, 2); allocator.schedule(); - rmDispatcher.await(); + rm.drainEvents(); amNodeManager.nodeHeartbeat(true); - rmDispatcher.await(); + rm.drainEvents(); allocator.schedule(); - rmDispatcher.await(); + rm.drainEvents(); // Wait for all reduce-tasks to be running for (Task t : job.getTasks().values()) { @@ -1013,14 +990,14 @@ public class TestRMContainerAllocator { finishNextNTasks(rmDispatcher, amNodeManager, mrApp, it, 2); allocator.schedule(); - rmDispatcher.await(); + rm.drainEvents(); Assert.assertEquals(0.59f, job.getProgress(), 0.001f); Assert.assertEquals(0.59f, rmApp.getProgress(), 0.001f); // Finish off the remaining 8 reduces. finishNextNTasks(rmDispatcher, amNodeManager, mrApp, it, 8); allocator.schedule(); - rmDispatcher.await(); + rm.drainEvents(); // Remaining is JobCleanup Assert.assertEquals(0.95f, job.getProgress(), 0.001f); Assert.assertEquals(0.95f, rmApp.getProgress(), 0.001f); @@ -1064,16 +1041,16 @@ public class TestRMContainerAllocator { // Submit the application RMApp rmApp = rm.submitApp(1024); - rmDispatcher.await(); + rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 11264); amNodeManager.nodeHeartbeat(true); - rmDispatcher.await(); + rm.drainEvents(); final ApplicationAttemptId appAttemptId = rmApp.getCurrentAppAttempt() .getAppAttemptId(); rm.sendAMLaunched(appAttemptId); - rmDispatcher.await(); + rm.drainEvents(); MRApp mrApp = new MRApp(appAttemptId, ContainerId.newContainerId( appAttemptId, 0), 10, 0, false, this.getClass().getName(), true, 1) { @@ -1109,11 +1086,11 @@ public class TestRMContainerAllocator { amDispatcher.await(); allocator.schedule(); - rmDispatcher.await(); + rm.drainEvents(); amNodeManager.nodeHeartbeat(true); - rmDispatcher.await(); + rm.drainEvents(); allocator.schedule(); - rmDispatcher.await(); + rm.drainEvents(); // Wait for all map-tasks to be running for (Task t : job.getTasks().values()) { @@ -1121,7 +1098,7 @@ public class TestRMContainerAllocator { } allocator.schedule(); // Send heartbeat - rmDispatcher.await(); + rm.drainEvents(); Assert.assertEquals(0.05f, job.getProgress(), 0.001f); Assert.assertEquals(0.05f, rmApp.getProgress(), 0.001f); @@ -1130,21 +1107,21 @@ public class TestRMContainerAllocator { // Finish off 1 map so that map-progress is 10% finishNextNTasks(rmDispatcher, amNodeManager, mrApp, it, 1); allocator.schedule(); - rmDispatcher.await(); + rm.drainEvents(); Assert.assertEquals(0.14f, job.getProgress(), 0.001f); Assert.assertEquals(0.14f, rmApp.getProgress(), 0.001f); // Finish off 5 more map so that map-progress is 60% finishNextNTasks(rmDispatcher, amNodeManager, mrApp, it, 5); allocator.schedule(); - rmDispatcher.await(); + rm.drainEvents(); Assert.assertEquals(0.59f, job.getProgress(), 0.001f); Assert.assertEquals(0.59f, rmApp.getProgress(), 0.001f); // Finish off remaining map so that map-progress is 100% finishNextNTasks(rmDispatcher, amNodeManager, mrApp, it, 4); allocator.schedule(); - rmDispatcher.await(); + rm.drainEvents(); Assert.assertEquals(0.95f, job.getProgress(), 0.001f); Assert.assertEquals(0.95f, rmApp.getProgress(), 0.001f); } @@ -1154,21 +1131,19 @@ public class TestRMContainerAllocator { Configuration conf = new Configuration(); MyResourceManager rm = new MyResourceManager(conf); rm.start(); - DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext() - .getDispatcher(); // Submit the application RMApp app = rm.submitApp(1024); - dispatcher.await(); + rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); amNodeManager.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt() .getAppAttemptId(); rm.sendAMLaunched(appAttemptId); - dispatcher.await(); - + rm.drainEvents(); + JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0); Job mockJob = mock(Job.class); MyContainerAllocator allocator = new MyContainerAllocator(rm, conf, @@ -1177,7 +1152,7 @@ public class TestRMContainerAllocator { // add resources to scheduler MockNM nm1 = rm.registerNode("h1:1234", 10240); MockNM nm2 = rm.registerNode("h2:1234", 10240); - dispatcher.await(); + rm.drainEvents(); // create the map container request ContainerRequestEvent event = createReq(jobId, 1, 1024, @@ -1193,16 +1168,16 @@ public class TestRMContainerAllocator { // this tells the scheduler about the requests List assigned = allocator.schedule(); - dispatcher.await(); + rm.drainEvents(); nm1.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); Assert.assertEquals(1, allocator.getJobUpdatedNodeEvents().size()); Assert.assertEquals(3, allocator.getJobUpdatedNodeEvents().get(0).getUpdatedNodes().size()); allocator.getJobUpdatedNodeEvents().clear(); // get the assignment assigned = allocator.schedule(); - dispatcher.await(); + rm.drainEvents(); Assert.assertEquals(1, assigned.size()); Assert.assertEquals(nm1.getNodeId(), assigned.get(0).getContainer().getNodeId()); // no updated nodes reported @@ -1212,11 +1187,11 @@ public class TestRMContainerAllocator { // mark nodes bad nm1.nodeHeartbeat(false); nm2.nodeHeartbeat(false); - dispatcher.await(); - + rm.drainEvents(); + // schedule response returns updated nodes assigned = allocator.schedule(); - dispatcher.await(); + rm.drainEvents(); Assert.assertEquals(0, assigned.size()); // updated nodes are reported Assert.assertEquals(1, allocator.getJobUpdatedNodeEvents().size()); @@ -1227,7 +1202,7 @@ public class TestRMContainerAllocator { allocator.getTaskAttemptKillEvents().clear(); assigned = allocator.schedule(); - dispatcher.await(); + rm.drainEvents(); Assert.assertEquals(0, assigned.size()); // no updated nodes reported Assert.assertTrue(allocator.getJobUpdatedNodeEvents().isEmpty()); @@ -1247,21 +1222,19 @@ public class TestRMContainerAllocator { MyResourceManager rm = new MyResourceManager(conf); rm.start(); - DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext() - .getDispatcher(); // Submit the application RMApp app = rm.submitApp(1024); - dispatcher.await(); + rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); amNodeManager.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt() .getAppAttemptId(); rm.sendAMLaunched(appAttemptId); - dispatcher.await(); + rm.drainEvents(); JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0); Job mockJob = mock(Job.class); @@ -1275,7 +1248,7 @@ public class TestRMContainerAllocator { MockNM nodeManager1 = rm.registerNode("h1:1234", 10240); MockNM nodeManager2 = rm.registerNode("h2:1234", 10240); MockNM nodeManager3 = rm.registerNode("h3:1234", 10240); - dispatcher.await(); + rm.drainEvents(); // create the container request ContainerRequestEvent event1 = createReq(jobId, 1, 1024, @@ -1295,7 +1268,7 @@ public class TestRMContainerAllocator { // this tells the scheduler about the requests // as nodes are not added, no allocations List assigned = allocator.schedule(); - dispatcher.await(); + rm.drainEvents(); Assert.assertEquals("No of assignments must be 0", 0, assigned.size()); // Send events to blacklist nodes h1 and h2 @@ -1307,28 +1280,28 @@ public class TestRMContainerAllocator { // update resources in scheduler nodeManager1.nodeHeartbeat(true); // Node heartbeat nodeManager2.nodeHeartbeat(true); // Node heartbeat - dispatcher.await(); + rm.drainEvents(); assigned = allocator.schedule(); Assert.assertEquals("No of assignments must be 0", 0, assigned.size()); - dispatcher.await(); + rm.drainEvents(); Assert.assertEquals("No of assignments must be 0", 0, assigned.size()); assertBlacklistAdditionsAndRemovals(2, 0, rm); // mark h1/h2 as bad nodes nodeManager1.nodeHeartbeat(false); nodeManager2.nodeHeartbeat(false); - dispatcher.await(); + rm.drainEvents(); assigned = allocator.schedule(); - dispatcher.await(); + rm.drainEvents(); assertBlacklistAdditionsAndRemovals(0, 0, rm); Assert.assertEquals("No of assignments must be 0", 0, assigned.size()); nodeManager3.nodeHeartbeat(true); // Node heartbeat - dispatcher.await(); + rm.drainEvents(); assigned = allocator.schedule(); - dispatcher.await(); + rm.drainEvents(); assertBlacklistAdditionsAndRemovals(0, 0, rm); Assert.assertTrue("No of assignments must be 3", assigned.size() == 3); @@ -1352,24 +1325,22 @@ public class TestRMContainerAllocator { MyResourceManager rm = new MyResourceManager(conf); rm.start(); - DrainDispatcher dispatcher = - (DrainDispatcher) rm.getRMContext().getDispatcher(); // Submit the application RMApp app = rm.submitApp(1024); - dispatcher.await(); + rm.drainEvents(); MockNM[] nodeManagers = new MockNM[10]; int nmNum = 0; List assigned = null; - nodeManagers[nmNum] = registerNodeManager(nmNum++, rm, dispatcher); + nodeManagers[nmNum] = registerNodeManager(nmNum++, rm); nodeManagers[0].nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt().getAppAttemptId(); rm.sendAMLaunched(appAttemptId); - dispatcher.await(); + rm.drainEvents(); JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0); Job mockJob = mock(Job.class); @@ -1382,7 +1353,7 @@ public class TestRMContainerAllocator { // Known=1, blacklisted=0, ignore should be false - assign first container assigned = getContainerOnHost(jobId, 1, 1024, new String[] { "h1" }, - nodeManagers[0], dispatcher, allocator, 0, 0, 0, 0, rm); + nodeManagers[0], allocator, 0, 0, 0, 0, rm); Assert.assertEquals("No of assignments must be 1", 1, assigned.size()); LOG.info("Failing container _1 on H1 (Node should be blacklisted and" @@ -1397,47 +1368,47 @@ public class TestRMContainerAllocator { // The current call will send blacklisted node "h1" to RM assigned = getContainerOnHost(jobId, 2, 1024, new String[] { "h1" }, - nodeManagers[0], dispatcher, allocator, 1, 0, 0, 1, rm); + nodeManagers[0], allocator, 1, 0, 0, 1, rm); Assert.assertEquals("No of assignments must be 0", 0, assigned.size()); // Known=1, blacklisted=1, ignore should be true - assign 1 assigned = getContainerOnHost(jobId, 2, 1024, new String[] { "h1" }, - nodeManagers[0], dispatcher, allocator, 0, 0, 0, 0, rm); + nodeManagers[0], allocator, 0, 0, 0, 0, rm); Assert.assertEquals("No of assignments must be 1", 1, assigned.size()); - nodeManagers[nmNum] = registerNodeManager(nmNum++, rm, dispatcher); + nodeManagers[nmNum] = registerNodeManager(nmNum++, rm); // Known=2, blacklisted=1, ignore should be true - assign 1 anyway. assigned = getContainerOnHost(jobId, 3, 1024, new String[] { "h2" }, - nodeManagers[1], dispatcher, allocator, 0, 0, 0, 0, rm); + nodeManagers[1], allocator, 0, 0, 0, 0, rm); Assert.assertEquals("No of assignments must be 1", 1, assigned.size()); - nodeManagers[nmNum] = registerNodeManager(nmNum++, rm, dispatcher); + nodeManagers[nmNum] = registerNodeManager(nmNum++, rm); // Known=3, blacklisted=1, ignore should be true - assign 1 anyway. assigned = getContainerOnHost(jobId, 4, 1024, new String[] { "h3" }, - nodeManagers[2], dispatcher, allocator, 0, 0, 0, 0, rm); + nodeManagers[2], allocator, 0, 0, 0, 0, rm); Assert.assertEquals("No of assignments must be 1", 1, assigned.size()); // Known=3, blacklisted=1, ignore should be true - assign 1 assigned = getContainerOnHost(jobId, 5, 1024, new String[] { "h1" }, - nodeManagers[0], dispatcher, allocator, 0, 0, 0, 0, rm); + nodeManagers[0], allocator, 0, 0, 0, 0, rm); Assert.assertEquals("No of assignments must be 1", 1, assigned.size()); - nodeManagers[nmNum] = registerNodeManager(nmNum++, rm, dispatcher); + nodeManagers[nmNum] = registerNodeManager(nmNum++, rm); // Known=4, blacklisted=1, ignore should be false - assign 1 anyway assigned = getContainerOnHost(jobId, 6, 1024, new String[] { "h4" }, - nodeManagers[3], dispatcher, allocator, 0, 0, 1, 0, rm); + nodeManagers[3], allocator, 0, 0, 1, 0, rm); Assert.assertEquals("No of assignments must be 1", 1, assigned.size()); // Test blacklisting re-enabled. // Known=4, blacklisted=1, ignore should be false - no assignment on h1 assigned = getContainerOnHost(jobId, 7, 1024, new String[] { "h1" }, - nodeManagers[0], dispatcher, allocator, 0, 0, 0, 0, rm); + nodeManagers[0], allocator, 0, 0, 0, 0, rm); Assert.assertEquals("No of assignments must be 0", 0, assigned.size()); // RMContainerRequestor would have created a replacement request. @@ -1450,61 +1421,61 @@ public class TestRMContainerAllocator { // container for the same reason above. assigned = getContainerOnHost(jobId, 8, 1024, new String[] { "h1" }, - nodeManagers[0], dispatcher, allocator, 1, 0, 0, 2, rm); + nodeManagers[0], allocator, 1, 0, 0, 2, rm); Assert.assertEquals("No of assignments must be 0", 0, assigned.size()); // Known=4, blacklisted=2, ignore should be true. Should assign 2 // containers. assigned = getContainerOnHost(jobId, 8, 1024, new String[] { "h1" }, - nodeManagers[0], dispatcher, allocator, 0, 0, 0, 0, rm); + nodeManagers[0], allocator, 0, 0, 0, 0, rm); Assert.assertEquals("No of assignments must be 2", 2, assigned.size()); // Known=4, blacklisted=2, ignore should be true. assigned = getContainerOnHost(jobId, 9, 1024, new String[] { "h2" }, - nodeManagers[1], dispatcher, allocator, 0, 0, 0, 0, rm); + nodeManagers[1], allocator, 0, 0, 0, 0, rm); Assert.assertEquals("No of assignments must be 1", 1, assigned.size()); // Test blacklist while ignore blacklisting enabled ContainerFailedEvent f3 = createFailEvent(jobId, 4, "h3", false); allocator.sendFailure(f3); - nodeManagers[nmNum] = registerNodeManager(nmNum++, rm, dispatcher); + nodeManagers[nmNum] = registerNodeManager(nmNum++, rm); // Known=5, blacklisted=3, ignore should be true. assigned = getContainerOnHost(jobId, 10, 1024, new String[] { "h3" }, - nodeManagers[2], dispatcher, allocator, 0, 0, 0, 0, rm); + nodeManagers[2], allocator, 0, 0, 0, 0, rm); Assert.assertEquals("No of assignments must be 1", 1, assigned.size()); // Assign on 5 more nodes - to re-enable blacklisting for (int i = 0; i < 5; i++) { - nodeManagers[nmNum] = registerNodeManager(nmNum++, rm, dispatcher); + nodeManagers[nmNum] = registerNodeManager(nmNum++, rm); assigned = getContainerOnHost(jobId, 11 + i, 1024, new String[] { String.valueOf(5 + i) }, nodeManagers[4 + i], - dispatcher, allocator, 0, 0, (i == 4 ? 3 : 0), 0, rm); + allocator, 0, 0, (i == 4 ? 3 : 0), 0, rm); Assert.assertEquals("No of assignments must be 1", 1, assigned.size()); } // Test h3 (blacklisted while ignoring blacklisting) is blacklisted. assigned = getContainerOnHost(jobId, 20, 1024, new String[] { "h3" }, - nodeManagers[2], dispatcher, allocator, 0, 0, 0, 0, rm); + nodeManagers[2], allocator, 0, 0, 0, 0, rm); Assert.assertEquals("No of assignments must be 0", 0, assigned.size()); } - private MockNM registerNodeManager(int i, MyResourceManager rm, - DrainDispatcher dispatcher) throws Exception { + private MockNM registerNodeManager(int i, MyResourceManager rm) + throws Exception { MockNM nm = rm.registerNode("h" + (i + 1) + ":1234", 10240); - dispatcher.await(); + rm.drainEvents(); return nm; } private List getContainerOnHost(JobId jobId, int taskAttemptId, int memory, String[] hosts, MockNM mockNM, - DrainDispatcher dispatcher, MyContainerAllocator allocator, + MyContainerAllocator allocator, int expectedAdditions1, int expectedRemovals1, int expectedAdditions2, int expectedRemovals2, MyResourceManager rm) throws Exception { @@ -1514,17 +1485,17 @@ public class TestRMContainerAllocator { // Send the request to the RM List assigned = allocator.schedule(); - dispatcher.await(); + rm.drainEvents(); assertBlacklistAdditionsAndRemovals( expectedAdditions1, expectedRemovals1, rm); Assert.assertEquals("No of assignments must be 0", 0, assigned.size()); // Heartbeat from the required nodeManager mockNM.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); assigned = allocator.schedule(); - dispatcher.await(); + rm.drainEvents(); assertBlacklistAdditionsAndRemovals( expectedAdditions2, expectedRemovals2, rm); return assigned; @@ -1542,21 +1513,19 @@ public class TestRMContainerAllocator { MyResourceManager rm = new MyResourceManager(conf); rm.start(); - DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext() - .getDispatcher(); // Submit the application RMApp app = rm.submitApp(1024); - dispatcher.await(); + rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); amNodeManager.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt() .getAppAttemptId(); rm.sendAMLaunched(appAttemptId); - dispatcher.await(); + rm.drainEvents(); JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0); Job mockJob = mock(Job.class); @@ -1569,7 +1538,7 @@ public class TestRMContainerAllocator { // add resources to scheduler MockNM nodeManager1 = rm.registerNode("h1:1234", 10240); MockNM nodeManager3 = rm.registerNode("h3:1234", 10240); - dispatcher.await(); + rm.drainEvents(); LOG.info("Requesting 1 Containers _1 on H1"); // create the container request @@ -1581,17 +1550,17 @@ public class TestRMContainerAllocator { // this tells the scheduler about the requests // as nodes are not added, no allocations List assigned = allocator.schedule(); - dispatcher.await(); + rm.drainEvents(); Assert.assertEquals("No of assignments must be 0", 0, assigned.size()); LOG.info("h1 Heartbeat (To actually schedule the containers)"); // update resources in scheduler nodeManager1.nodeHeartbeat(true); // Node heartbeat - dispatcher.await(); + rm.drainEvents(); LOG.info("RM Heartbeat (To process the scheduled containers)"); assigned = allocator.schedule(); - dispatcher.await(); + rm.drainEvents(); assertBlacklistAdditionsAndRemovals(0, 0, rm); Assert.assertEquals("No of assignments must be 1", 1, assigned.size()); @@ -1608,7 +1577,7 @@ public class TestRMContainerAllocator { //Update the Scheduler with the new requests. assigned = allocator.schedule(); - dispatcher.await(); + rm.drainEvents(); assertBlacklistAdditionsAndRemovals(1, 0, rm); Assert.assertEquals("No of assignments must be 0", 0, assigned.size()); @@ -1623,11 +1592,11 @@ public class TestRMContainerAllocator { LOG.info("h1 Heartbeat (To actually schedule the containers)"); // update resources in scheduler nodeManager1.nodeHeartbeat(true); // Node heartbeat - dispatcher.await(); + rm.drainEvents(); LOG.info("RM Heartbeat (To process the scheduled containers)"); assigned = allocator.schedule(); - dispatcher.await(); + rm.drainEvents(); assertBlacklistAdditionsAndRemovals(0, 0, rm); Assert.assertEquals("No of assignments must be 0", 0, assigned.size()); @@ -1636,19 +1605,19 @@ public class TestRMContainerAllocator { //Send a release for the p:5 container + another request. LOG.info("RM Heartbeat (To process the re-scheduled containers)"); assigned = allocator.schedule(); - dispatcher.await(); + rm.drainEvents(); assertBlacklistAdditionsAndRemovals(0, 0, rm); Assert.assertEquals("No of assignments must be 0", 0, assigned.size()); //Hearbeat from H3 to schedule on this host. LOG.info("h3 Heartbeat (To re-schedule the containers)"); nodeManager3.nodeHeartbeat(true); // Node heartbeat - dispatcher.await(); + rm.drainEvents(); LOG.info("RM Heartbeat (To process the re-scheduled containers for H3)"); assigned = allocator.schedule(); assertBlacklistAdditionsAndRemovals(0, 0, rm); - dispatcher.await(); + rm.drainEvents(); // For debugging for (TaskAttemptContainerAssignedEvent assig : assigned) { @@ -2229,22 +2198,20 @@ public class TestRMContainerAllocator { Configuration conf = new Configuration(); final MyResourceManager rm = new MyResourceManager(conf); rm.start(); - DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext() - .getDispatcher(); // Submit the application RMApp app = rm.submitApp(1024); - dispatcher.await(); + rm.drainEvents(); // Make a node to register so as to launch the AM. MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); amNodeManager.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt() .getAppAttemptId(); rm.sendAMLaunched(appAttemptId); - dispatcher.await(); + rm.drainEvents(); JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0); Job job = mock(Job.class); @@ -2381,21 +2348,19 @@ public class TestRMContainerAllocator { Configuration conf = new Configuration(); final MyResourceManager rm = new MyResourceManager(conf); rm.start(); - DrainDispatcher rmDispatcher = - (DrainDispatcher) rm.getRMContext().getDispatcher(); // Submit the application RMApp rmApp = rm.submitApp(1024); - rmDispatcher.await(); + rm.drainEvents(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 11264); amNodeManager.nodeHeartbeat(true); - rmDispatcher.await(); + rm.drainEvents(); final ApplicationAttemptId appAttemptId = rmApp.getCurrentAppAttempt().getAppAttemptId(); rm.sendAMLaunched(appAttemptId); - rmDispatcher.await(); + rm.drainEvents(); MRApp mrApp = new MRApp(appAttemptId, ContainerId.newContainerId(appAttemptId, 0), 10, @@ -2454,22 +2419,20 @@ public class TestRMContainerAllocator { MyResourceManager rm1 = new MyResourceManager(conf, memStore); rm1.start(); - DrainDispatcher dispatcher = - (DrainDispatcher) rm1.getRMContext().getDispatcher(); // Submit the application RMApp app = rm1.submitApp(1024); - dispatcher.await(); + rm1.drainEvents(); MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService()); nm1.registerNode(); nm1.nodeHeartbeat(true); // Node heartbeat - dispatcher.await(); + rm1.drainEvents(); ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt().getAppAttemptId(); rm1.sendAMLaunched(appAttemptId); - dispatcher.await(); + rm1.drainEvents(); JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0); Job mockJob = mock(Job.class); @@ -2498,7 +2461,7 @@ public class TestRMContainerAllocator { // send allocate request and 1 blacklisted nodes List assignedContainers = allocator.schedule(); - dispatcher.await(); + rm1.drainEvents(); Assert.assertEquals("No of assignments must be 0", 0, assignedContainers.size()); // Why ask is 3, not 4? --> ask from blacklisted node h2 is removed @@ -2506,11 +2469,11 @@ public class TestRMContainerAllocator { assertBlacklistAdditionsAndRemovals(1, 0, rm1); nm1.nodeHeartbeat(true); // Node heartbeat - dispatcher.await(); + rm1.drainEvents(); // Step-2 : 2 containers are allocated by RM. assignedContainers = allocator.schedule(); - dispatcher.await(); + rm1.drainEvents(); Assert.assertEquals("No of assignments must be 2", 2, assignedContainers.size()); assertAsksAndReleases(0, 0, rm1); @@ -2545,7 +2508,6 @@ public class TestRMContainerAllocator { rm2.start(); nm1.setResourceTrackerService(rm2.getResourceTrackerService()); allocator.updateSchedulerProxy(rm2); - dispatcher = (DrainDispatcher) rm2.getRMContext().getDispatcher(); // NM should be rebooted on heartbeat, even first heartbeat for nm2 NodeHeartbeatResponse hbResponse = nm1.nodeHeartbeat(true); @@ -2555,7 +2517,7 @@ public class TestRMContainerAllocator { nm1 = new MockNM("h1:1234", 10240, rm2.getResourceTrackerService()); nm1.registerNode(); nm1.nodeHeartbeat(true); - dispatcher.await(); + rm2.drainEvents(); // Step-4 : On RM restart, AM(does not know RM is restarted) sends // additional containerRequest(event4) and blacklisted nodes. @@ -2576,7 +2538,7 @@ public class TestRMContainerAllocator { // send allocate request to 2nd RM and get resync command allocator.schedule(); - dispatcher.await(); + rm2.drainEvents(); // Step-5 : On Resync,AM sends all outstanding // asks,release,blacklistAaddition @@ -2587,16 +2549,16 @@ public class TestRMContainerAllocator { // send all outstanding request again. assignedContainers = allocator.schedule(); - dispatcher.await(); + rm2.drainEvents(); assertAsksAndReleases(3, 2, rm2); assertBlacklistAdditionsAndRemovals(2, 0, rm2); nm1.nodeHeartbeat(true); - dispatcher.await(); + rm2.drainEvents(); // Step-6 : RM allocates containers i.e event3,event4 and cRequest5 assignedContainers = allocator.schedule(); - dispatcher.await(); + rm2.drainEvents(); Assert.assertEquals("Number of container should be 3", 3, assignedContainers.size()); @@ -2699,20 +2661,19 @@ public class TestRMContainerAllocator { MRJobConfig.MR_AM_TO_RM_WAIT_INTERVAL_MS, 0); MyResourceManager rm1 = new MyResourceManager(conf); rm1.start(); - DrainDispatcher dispatcher = - (DrainDispatcher) rm1.getRMContext().getDispatcher(); + RMApp app = rm1.submitApp(1024); - dispatcher.await(); + rm1.drainEvents(); MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService()); nm1.registerNode(); nm1.nodeHeartbeat(true); - dispatcher.await(); + rm1.drainEvents(); ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt().getAppAttemptId(); rm1.sendAMLaunched(appAttemptId); - dispatcher.await(); + rm1.drainEvents(); JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0); Job mockJob = mock(Job.class); @@ -2728,7 +2689,7 @@ public class TestRMContainerAllocator { } catch (RMContainerAllocationException e) { Assert.assertTrue(e.getMessage().contains("Could not contact RM after")); } - dispatcher.await(); + rm1.drainEvents(); Assert.assertEquals("Should Have 1 Job Event", 1, allocator.jobEvents.size()); JobEvent event = allocator.jobEvents.get(0); @@ -2750,22 +2711,20 @@ public class TestRMContainerAllocator { rm.start(); AMRMTokenSecretManager secretMgr = rm.getRMContext().getAMRMTokenSecretManager(); - DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext() - .getDispatcher(); // Submit the application RMApp app = rm.submitApp(1024); - dispatcher.await(); + rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); amNodeManager.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); final ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt() .getAppAttemptId(); final ApplicationId appId = app.getApplicationId(); rm.sendAMLaunched(appAttemptId); - dispatcher.await(); + rm.drainEvents(); JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0); final Job mockJob = mock(Job.class); @@ -2953,21 +2912,19 @@ public class TestRMContainerAllocator { Configuration conf = new Configuration(); MyResourceManager rm = new MyResourceManager(conf); rm.start(); - DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext() - .getDispatcher(); // Submit the application RMApp app = rm.submitApp(1024); - dispatcher.await(); + rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); amNodeManager.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt() .getAppAttemptId(); rm.sendAMLaunched(appAttemptId); - dispatcher.await(); + rm.drainEvents(); JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0); Job mockJob = mock(Job.class); @@ -2989,21 +2946,19 @@ public class TestRMContainerAllocator { Configuration conf = new Configuration(); MyResourceManager rm = new MyResourceManager(conf); rm.start(); - DrainDispatcher dispatcher = - (DrainDispatcher) rm.getRMContext().getDispatcher(); // Submit the application RMApp app = rm.submitApp(1024); - dispatcher.await(); + rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 1260); amNodeManager.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt() .getAppAttemptId(); rm.sendAMLaunched(appAttemptId); - dispatcher.await(); + rm.drainEvents(); JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0); Job mockJob = mock(Job.class); @@ -3018,7 +2973,7 @@ public class TestRMContainerAllocator { // Register nodes to RM. MockNM nodeManager = rm.registerNode("h1:1234", 1024); - dispatcher.await(); + rm.drainEvents(); // Request 2 maps and 1 reducer(sone on nodes which are not registered). ContainerRequestEvent event1 = @@ -3034,7 +2989,7 @@ public class TestRMContainerAllocator { // This will tell the scheduler about the requests but there will be no // allocations as nodes are not added. allocator.schedule(); - dispatcher.await(); + rm.drainEvents(); // Advance clock so that maps can be considered as hanging. clock.setTime(System.currentTimeMillis() + 500000L); @@ -3045,15 +3000,15 @@ public class TestRMContainerAllocator { allocator.sendRequest(event4); allocator.schedule(); - dispatcher.await(); + rm.drainEvents(); // Update resources in scheduler through node heartbeat from h1. nodeManager.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); rm.getMyFifoScheduler().forceResourceLimit(Resource.newInstance(1024, 1)); allocator.schedule(); - dispatcher.await(); + rm.drainEvents(); // One map is assigned. Assert.assertEquals(1, allocator.getAssignedRequests().maps.size()); @@ -3087,7 +3042,7 @@ public class TestRMContainerAllocator { // On next allocate request to scheduler, headroom reported will be 0. rm.getMyFifoScheduler().forceResourceLimit(Resource.newInstance(0, 0)); allocator.schedule(); - dispatcher.await(); + rm.drainEvents(); // After allocate response from scheduler, all scheduled reduces are ramped // down and move to pending. 3 asks are also updated with 0 containers to // indicate ramping down of reduces to scheduler. @@ -3155,21 +3110,19 @@ public class TestRMContainerAllocator { Configuration conf = new Configuration(); MyResourceManager rm = new MyResourceManager(conf); rm.start(); - DrainDispatcher dispatcher = - (DrainDispatcher) rm.getRMContext().getDispatcher(); // Submit the application RMApp app = rm.submitApp(1024); - dispatcher.await(); + rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 1260); amNodeManager.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt() .getAppAttemptId(); rm.sendAMLaunched(appAttemptId); - dispatcher.await(); + rm.drainEvents(); JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0); Job mockJob = mock(Job.class); @@ -3184,7 +3137,7 @@ public class TestRMContainerAllocator { // Register nodes to RM. MockNM nodeManager = rm.registerNode("h1:1234", 1024); - dispatcher.await(); + rm.drainEvents(); // Request 2 maps and 1 reducer(sone on nodes which are not registered). ContainerRequestEvent event1 = @@ -3200,7 +3153,7 @@ public class TestRMContainerAllocator { // This will tell the scheduler about the requests but there will be no // allocations as nodes are not added. allocator.schedule(); - dispatcher.await(); + rm.drainEvents(); // Advance clock so that maps can be considered as hanging. clock.setTime(System.currentTimeMillis() + 500000L); @@ -3211,15 +3164,15 @@ public class TestRMContainerAllocator { allocator.sendRequest(event4); allocator.schedule(); - dispatcher.await(); + rm.drainEvents(); // Update resources in scheduler through node heartbeat from h1. nodeManager.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); rm.getMyFifoScheduler().forceResourceLimit(Resource.newInstance(1024, 1)); allocator.schedule(); - dispatcher.await(); + rm.drainEvents(); // One map is assigned. Assert.assertEquals(1, allocator.getAssignedRequests().maps.size()); @@ -3256,7 +3209,7 @@ public class TestRMContainerAllocator { // On next allocate request to scheduler, headroom reported will be 2048. rm.getMyFifoScheduler().forceResourceLimit(Resource.newInstance(2048, 0)); allocator.schedule(); - dispatcher.await(); + rm.drainEvents(); // After allocate response from scheduler, all scheduled reduces are ramped // down and move to pending. 3 asks are also updated with 0 containers to // indicate ramping down of reduces to scheduler. @@ -3285,21 +3238,19 @@ public class TestRMContainerAllocator { conf.setInt(MRJobConfig.MR_JOB_REDUCER_UNCONDITIONAL_PREEMPT_DELAY_SEC, -1); MyResourceManager rm = new MyResourceManager(conf); rm.start(); - DrainDispatcher dispatcher = - (DrainDispatcher) rm.getRMContext().getDispatcher(); // Submit the application RMApp app = rm.submitApp(1024); - dispatcher.await(); + rm.drainEvents(); MockNM amNodeManager = rm.registerNode("amNM:1234", 1260); amNodeManager.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt() .getAppAttemptId(); rm.sendAMLaunched(appAttemptId); - dispatcher.await(); + rm.drainEvents(); JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0); Job mockJob = mock(Job.class); @@ -3315,10 +3266,10 @@ public class TestRMContainerAllocator { appAttemptId, mockJob); MockNM nodeManager = rm.registerNode("h1:1234", 4096); - dispatcher.await(); + rm.drainEvents(); // Register nodes to RM. MockNM nodeManager2 = rm.registerNode("h2:1234", 1024); - dispatcher.await(); + rm.drainEvents(); // Request 2 maps and 1 reducer(sone on nodes which are not registered). ContainerRequestEvent event1 = @@ -3334,7 +3285,7 @@ public class TestRMContainerAllocator { // This will tell the scheduler about the requests but there will be no // allocations as nodes are not added. allocator.schedule(); - dispatcher.await(); + rm.drainEvents(); // Request for another reducer on h3 which has not registered. ContainerRequestEvent event4 = @@ -3342,15 +3293,15 @@ public class TestRMContainerAllocator { allocator.sendRequest(event4); allocator.schedule(); - dispatcher.await(); + rm.drainEvents(); // Update resources in scheduler through node heartbeat from h1. nodeManager.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); rm.getMyFifoScheduler().forceResourceLimit(Resource.newInstance(3072, 3)); allocator.schedule(); - dispatcher.await(); + rm.drainEvents(); // Two maps are assigned. Assert.assertEquals(2, allocator.getAssignedRequests().maps.size()); @@ -3363,15 +3314,15 @@ public class TestRMContainerAllocator { Assert.assertEquals(0, allocator.getAssignedRequests().maps.size()); nodeManager.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); rm.getMyFifoScheduler().forceResourceLimit(Resource.newInstance(1024, 1)); allocator.schedule(); - dispatcher.await(); + rm.drainEvents(); // h2 heartbeats. nodeManager2.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); // Send request for one more mapper. ContainerRequestEvent event5 = @@ -3380,7 +3331,7 @@ public class TestRMContainerAllocator { rm.getMyFifoScheduler().forceResourceLimit(Resource.newInstance(2048, 2)); allocator.schedule(); - dispatcher.await(); + rm.drainEvents(); // One reducer is assigned and one map is scheduled Assert.assertEquals(1, allocator.getScheduledRequests().maps.size()); Assert.assertEquals(1, allocator.getAssignedRequests().reduces.size()); @@ -3388,7 +3339,7 @@ public class TestRMContainerAllocator { // enough if scheduled reducers resources are deducted. rm.getMyFifoScheduler().forceResourceLimit(Resource.newInstance(1260, 2)); allocator.schedule(); - dispatcher.await(); + rm.drainEvents(); // After allocate response, the one assigned reducer is preempted and killed Assert.assertEquals(1, MyContainerAllocator.getTaskAttemptKillEvents().size()); Assert.assertEquals(RMContainerAllocator.RAMPDOWN_DIAGNOSTIC, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java index 39a76333ab8..fa3c6afeb36 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java @@ -49,8 +49,6 @@ import org.apache.hadoop.yarn.api.records.UpdateContainerRequest; import org.apache.hadoop.yarn.client.api.AMRMClient; import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.event.Dispatcher; -import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; @@ -126,22 +124,20 @@ public class TestAMRMClientOnRMRestart { // Phase-1 Start 1st RM MyResourceManager rm1 = new MyResourceManager(conf, memStore); rm1.start(); - DrainDispatcher dispatcher = - (DrainDispatcher) rm1.getRMContext().getDispatcher(); // Submit the application RMApp app = rm1.submitApp(1024); - dispatcher.await(); + rm1.drainEvents(); MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService()); nm1.registerNode(); nm1.nodeHeartbeat(true); // Node heartbeat - dispatcher.await(); + rm1.drainEvents(); ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt().getAppAttemptId(); rm1.sendAMLaunched(appAttemptId); - dispatcher.await(); + rm1.drainEvents(); org.apache.hadoop.security.token.Token token = rm1.getRMContext().getRMApps().get(appAttemptId.getApplicationId()) @@ -176,7 +172,7 @@ public class TestAMRMClientOnRMRestart { blacklistAdditions.remove("h2");// remove from local list AllocateResponse allocateResponse = amClient.allocate(0.1f); - dispatcher.await(); + rm1.drainEvents(); Assert.assertEquals("No of assignments must be 0", 0, allocateResponse .getAllocatedContainers().size()); @@ -189,10 +185,10 @@ public class TestAMRMClientOnRMRestart { // Step-2 : NM heart beat is sent. // On 2nd AM allocate request, RM allocates 3 containers to AM nm1.nodeHeartbeat(true); // Node heartbeat - dispatcher.await(); + rm1.drainEvents(); allocateResponse = amClient.allocate(0.2f); - dispatcher.await(); + rm1.drainEvents(); // 3 containers are allocated i.e for cRequest1, cRequest2 and cRequest3. Assert.assertEquals("No of assignments must be 0", 3, allocateResponse .getAllocatedContainers().size()); @@ -207,7 +203,7 @@ public class TestAMRMClientOnRMRestart { amClient.removeContainerRequest(cRequest3); allocateResponse = amClient.allocate(0.2f); - dispatcher.await(); + rm1.drainEvents(); Assert.assertEquals("No of assignments must be 0", 0, allocateResponse .getAllocatedContainers().size()); assertAsksAndReleases(4, 0, rm1); @@ -233,7 +229,7 @@ public class TestAMRMClientOnRMRestart { // request nm1.nodeHeartbeat(containerId.getApplicationAttemptId(), containerId.getContainerId(), ContainerState.RUNNING); - dispatcher.await(); + rm1.drainEvents(); amClient.requestContainerUpdate( container, UpdateContainerRequest.newInstance( container.getVersion(), container.getId(), @@ -242,7 +238,7 @@ public class TestAMRMClientOnRMRestart { it.remove(); allocateResponse = amClient.allocate(0.3f); - dispatcher.await(); + rm1.drainEvents(); Assert.assertEquals("No of assignments must be 0", 0, allocateResponse .getAllocatedContainers().size()); assertAsksAndReleases(3, pendingRelease, rm1); @@ -258,7 +254,6 @@ public class TestAMRMClientOnRMRestart { rm2.start(); nm1.setResourceTrackerService(rm2.getResourceTrackerService()); ((MyAMRMClientImpl) amClient).updateRMProxy(rm2); - dispatcher = (DrainDispatcher) rm2.getRMContext().getDispatcher(); // NM should be rebooted on heartbeat, even first heartbeat for nm2 NodeHeartbeatResponse hbResponse = nm1.nodeHeartbeat(true); @@ -274,7 +269,7 @@ public class TestAMRMClientOnRMRestart { Collections.singletonList( containerId.getApplicationAttemptId().getApplicationId())); nm1.nodeHeartbeat(true); - dispatcher.await(); + rm2.drainEvents(); blacklistAdditions.add("h3"); amClient.updateBlacklist(blacklistAdditions, null); @@ -296,7 +291,7 @@ public class TestAMRMClientOnRMRestart { // containerRequest and blacklisted nodes. // Intern RM send resync command,AMRMClient resend allocate request allocateResponse = amClient.allocate(0.3f); - dispatcher.await(); + rm2.drainEvents(); completedContainer = allocateResponse.getCompletedContainersStatuses().size(); @@ -313,7 +308,7 @@ public class TestAMRMClientOnRMRestart { // Step-5 : Allocater after resync command allocateResponse = amClient.allocate(0.5f); - dispatcher.await(); + rm2.drainEvents(); Assert.assertEquals("No of assignments must be 0", 0, allocateResponse .getAllocatedContainers().size()); @@ -326,10 +321,10 @@ public class TestAMRMClientOnRMRestart { int count = 5; while (count-- > 0) { nm1.nodeHeartbeat(true); - dispatcher.await(); + rm2.drainEvents(); allocateResponse = amClient.allocate(0.5f); - dispatcher.await(); + rm2.drainEvents(); noAssignedContainer += allocateResponse.getAllocatedContainers().size(); if (noAssignedContainer == 3) { break; @@ -358,22 +353,20 @@ public class TestAMRMClientOnRMRestart { // Phase-1 Start 1st RM MyResourceManager rm1 = new MyResourceManager(conf, memStore); rm1.start(); - DrainDispatcher dispatcher = - (DrainDispatcher) rm1.getRMContext().getDispatcher(); // Submit the application RMApp app = rm1.submitApp(1024); - dispatcher.await(); + rm1.drainEvents(); MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService()); nm1.registerNode(); nm1.nodeHeartbeat(true); // Node heartbeat - dispatcher.await(); + rm1.drainEvents(); ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt().getAppAttemptId(); rm1.sendAMLaunched(appAttemptId); - dispatcher.await(); + rm1.drainEvents(); org.apache.hadoop.security.token.Token token = rm1.getRMContext().getRMApps().get(appAttemptId.getApplicationId()) @@ -393,7 +386,6 @@ public class TestAMRMClientOnRMRestart { rm2.start(); nm1.setResourceTrackerService(rm2.getResourceTrackerService()); ((MyAMRMClientImpl) amClient).updateRMProxy(rm2); - dispatcher = (DrainDispatcher) rm2.getRMContext().getDispatcher(); // NM should be rebooted on heartbeat, even first heartbeat for nm2 NodeHeartbeatResponse hbResponse = nm1.nodeHeartbeat(true); @@ -409,7 +401,7 @@ public class TestAMRMClientOnRMRestart { Priority.newInstance(0), 0); nm1.registerNode(Arrays.asList(containerReport), null); nm1.nodeHeartbeat(true); - dispatcher.await(); + rm2.drainEvents(); amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, null, null); @@ -421,7 +413,6 @@ public class TestAMRMClientOnRMRestart { amClient.stop(); rm1.stop(); rm2.stop(); - } @@ -439,22 +430,20 @@ public class TestAMRMClientOnRMRestart { // start first RM MyResourceManager2 rm1 = new MyResourceManager2(conf, memStore); rm1.start(); - DrainDispatcher dispatcher = - (DrainDispatcher) rm1.getRMContext().getDispatcher(); Long startTime = System.currentTimeMillis(); // Submit the application RMApp app = rm1.submitApp(1024); - dispatcher.await(); + rm1.drainEvents(); MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService()); nm1.registerNode(); nm1.nodeHeartbeat(true); // Node heartbeat - dispatcher.await(); + rm1.drainEvents(); ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt().getAppAttemptId(); rm1.sendAMLaunched(appAttemptId); - dispatcher.await(); + rm1.drainEvents(); AMRMTokenSecretManager amrmTokenSecretManagerForRM1 = rm1.getRMContext().getAMRMTokenSecretManager(); @@ -513,7 +502,6 @@ public class TestAMRMClientOnRMRestart { rm2.start(); nm1.setResourceTrackerService(rm2.getResourceTrackerService()); ((MyAMRMClientImpl) amClient).updateRMProxy(rm2); - dispatcher = (DrainDispatcher) rm2.getRMContext().getDispatcher(); AMRMTokenSecretManager amrmTokenSecretManagerForRM2 = rm2.getRMContext().getAMRMTokenSecretManager(); @@ -615,11 +603,6 @@ public class TestAMRMClientOnRMRestart { MyResourceManager.setClusterTimeStamp(fakeClusterTimeStamp); } - @Override - protected Dispatcher createDispatcher() { - return new DrainDispatcher(); - } - @Override protected EventHandler createSchedulerEventDispatcher() { // Dispatch inline for test sanity diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java index fbd5ac38dc0..100eb7f21a9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java @@ -30,14 +30,9 @@ import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.service.Service.STATE; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.event.Dispatcher; -import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.junit.Before; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - public abstract class ACLsTestBase { protected static final String COMMON_USER = "common_user"; @@ -80,11 +75,6 @@ public abstract class ACLsTestBase { .getRMDelegationTokenSecretManager()); } - @Override - protected Dispatcher createDispatcher() { - return new DrainDispatcher(); - } - @Override protected void doSecureLogin() throws IOException { } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java index c9ce7d7a061..c95bcdfca97 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java @@ -26,22 +26,17 @@ import org.apache.hadoop.ha.HAServiceProtocol; import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; import org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; -import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.event.Dispatcher; -import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; -import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -108,20 +103,9 @@ public abstract class RMHATestBase extends ClientBaseWithFixes{ } protected void startRMs() throws IOException { - rm1 = new MockRM(confForRM1, null, false, false){ - @Override - protected Dispatcher createDispatcher() { - return new DrainDispatcher(); - } - }; - rm2 = new MockRM(confForRM2, null, false, false){ - @Override - protected Dispatcher createDispatcher() { - return new DrainDispatcher(); - } - }; + rm1 = new MockRM(confForRM1, null, false, false); + rm2 = new MockRM(confForRM2, null, false, false); startRMs(rm1, confForRM1, rm2, confForRM2); - } protected void startRMsWithCustomizedRMAppManager() throws IOException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java index 03bc8897ab4..c8ee00e60bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java @@ -47,7 +47,6 @@ import org.apache.hadoop.yarn.api.records.ReservationRequest; import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter; import org.apache.hadoop.yarn.api.records.ReservationRequests; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; @@ -463,9 +462,7 @@ public class ReservationACLsTestBase extends ACLsTestBase { int attempts = 10; Collection plans; do { - DrainDispatcher dispatcher = - (DrainDispatcher) resourceManager.getRMContext().getDispatcher(); - dispatcher.await(); + resourceManager.drainEvents(); LOG.info("Waiting for node capacity to be added to plan"); plans = resourceManager.getRMContext().getReservationSystem() .getAllPlans().values(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java index c4197a1c1f8..422b7eb88a2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java @@ -40,8 +40,6 @@ import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.event.Dispatcher; -import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore; @@ -161,13 +159,7 @@ public class TestApplicationCleanup { Logger rootLogger = LogManager.getRootLogger(); rootLogger.setLevel(Level.DEBUG); - final DrainDispatcher dispatcher = new DrainDispatcher(); - MockRM rm = new MockRM() { - @Override - protected Dispatcher createDispatcher() { - return dispatcher; - } - }; + MockRM rm = new MockRM(); rm.start(); MockNM nm1 = rm.registerNode("127.0.0.1:1234", 5000); @@ -185,8 +177,8 @@ public class TestApplicationCleanup { int request = 2; am.allocate("127.0.0.1" , 1000, request, new ArrayList()); - dispatcher.await(); - + rm.drainEvents(); + //kick the scheduler nm1.nodeHeartbeat(true); List conts = am.allocate(new ArrayList(), @@ -199,7 +191,7 @@ public class TestApplicationCleanup { Thread.sleep(100); conts = am.allocate(new ArrayList(), new ArrayList()).getAllocatedContainers(); - dispatcher.await(); + rm.drainEvents(); contReceived += conts.size(); nm1.nodeHeartbeat(true); } @@ -209,7 +201,7 @@ public class TestApplicationCleanup { ArrayList release = new ArrayList(); release.add(conts.get(0).getId()); am.allocate(new ArrayList(), release); - dispatcher.await(); + rm.drainEvents(); // Send one more heartbeat with a fake running container. This is to // simulate the situation that can happen if the NM reports that container @@ -224,7 +216,7 @@ public class TestApplicationCleanup { containerStatuses.put(app.getApplicationId(), containerStatusList); NodeHeartbeatResponse resp = nm1.nodeHeartbeat(containerStatuses, true); - waitForContainerCleanup(dispatcher, nm1, resp); + waitForContainerCleanup(rm, nm1, resp); // Now to test the case when RM already gave cleanup, and NM suddenly // realizes that the container is running. @@ -240,17 +232,17 @@ public class TestApplicationCleanup { resp = nm1.nodeHeartbeat(containerStatuses, true); // The cleanup list won't be instantaneous as it is given out by scheduler // and not RMNodeImpl. - waitForContainerCleanup(dispatcher, nm1, resp); + waitForContainerCleanup(rm, nm1, resp); rm.stop(); } - protected void waitForContainerCleanup(DrainDispatcher dispatcher, MockNM nm, + protected void waitForContainerCleanup(MockRM rm, MockNM nm, NodeHeartbeatResponse resp) throws Exception { int waitCount = 0, cleanedConts = 0; List contsToClean; do { - dispatcher.await(); + rm.drainEvents(); contsToClean = resp.getContainersToCleanup(); cleanedConts += contsToClean.size(); if (cleanedConts >= 1) { @@ -400,13 +392,7 @@ public class TestApplicationCleanup { memStore.init(conf); // start RM - final DrainDispatcher dispatcher = new DrainDispatcher(); - MockRM rm1 = new MockRM(conf, memStore) { - @Override - protected Dispatcher createDispatcher() { - return dispatcher; - } - }; + MockRM rm1 = new MockRM(conf, memStore); rm1.start(); MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService()); @@ -419,13 +405,7 @@ public class TestApplicationCleanup { rm1.waitForState(app0.getApplicationId(), RMAppState.RUNNING); // start new RM - final DrainDispatcher dispatcher2 = new DrainDispatcher(); - MockRM rm2 = new MockRM(conf, memStore) { - @Override - protected Dispatcher createDispatcher() { - return dispatcher2; - } - }; + MockRM rm2 = new MockRM(conf, memStore); rm2.start(); // nm1 register to rm2, and do a heartbeat @@ -437,7 +417,7 @@ public class TestApplicationCleanup { NodeHeartbeatResponse response = nm1.nodeHeartbeat(am0 .getApplicationAttemptId(), 2, ContainerState.RUNNING); - waitForContainerCleanup(dispatcher2, nm1, response); + waitForContainerCleanup(rm2, nm1, response); rm1.stop(); rm2.stop(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java index 08b180fc413..9e8401027e9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java @@ -59,8 +59,6 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.SerializedException; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.event.Dispatcher; -import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException; import org.apache.hadoop.yarn.exceptions.ApplicationMasterNotRegisteredException; import org.apache.hadoop.yarn.exceptions.NMNotYetReadyException; @@ -260,7 +258,6 @@ public class TestApplicationMasterLauncher { Configuration conf = new Configuration(); conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1); conf.setInt(YarnConfiguration.CLIENT_NM_CONNECT_RETRY_INTERVAL_MS, 1); - final DrainDispatcher dispatcher = new DrainDispatcher(); MockRM rm = new MockRMWithCustomAMLauncher(conf, null) { @Override protected ApplicationMasterLauncher createAMLauncher() { @@ -284,12 +281,8 @@ public class TestApplicationMasterLauncher { } }; } - - @Override - protected Dispatcher createDispatcher() { - return dispatcher; - } }; + rm.start(); MockNM nm1 = rm.registerNode("127.0.0.1:1234", 5120); @@ -297,7 +290,7 @@ public class TestApplicationMasterLauncher { // kick the scheduling nm1.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); MockRM.waitForState(app.getCurrentAppAttempt(), RMAppAttemptState.LAUNCHED, 500); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java index 23bed228e19..18c49bdddcd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java @@ -42,8 +42,6 @@ import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.UpdateContainerRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.event.Dispatcher; -import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.exceptions.ApplicationMasterNotRegisteredException; import org.apache.hadoop.yarn.exceptions.InvalidContainerReleaseException; import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes; @@ -327,10 +325,8 @@ public class TestApplicationMasterService { @Test(timeout=1200000) public void testAllocateAfterUnregister() throws Exception { - MyResourceManager rm = new MyResourceManager(conf); + MockRM rm = new MockRM(conf); rm.start(); - DrainDispatcher rmDispatcher = (DrainDispatcher) rm.getRMContext() - .getDispatcher(); // Register node1 MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB); @@ -351,7 +347,7 @@ public class TestApplicationMasterService { AllocateResponse alloc1Response = am1.schedule(); nm1.nodeHeartbeat(true); - rmDispatcher.await(); + rm.drainEvents(); alloc1Response = am1.schedule(); Assert.assertEquals(0, alloc1Response.getAllocatedContainers().size()); } @@ -474,17 +470,6 @@ public class TestApplicationMasterService { rm.stop(); } - private static class MyResourceManager extends MockRM { - - public MyResourceManager(YarnConfiguration conf) { - super(conf); - } - @Override - protected Dispatcher createDispatcher() { - return new DrainDispatcher(); - } - } - private void sentRMContainerLaunched(MockRM rm, ContainerId containerId) { CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); RMContainer rmContainer = cs.getRMContainer(containerId); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java index b4adf480b35..75ef5c775be 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestNodeBlacklistingOnAMFailures.java @@ -33,8 +33,6 @@ import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.event.Dispatcher; -import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.TestAMRestart; import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -65,8 +63,7 @@ public class TestNodeBlacklistingOnAMFailures { conf.setBoolean(YarnConfiguration.AM_SCHEDULING_NODE_BLACKLISTING_ENABLED, true); - DrainDispatcher dispatcher = new DrainDispatcher(); - MockRM rm = startRM(conf, dispatcher); + MockRM rm = startRM(conf); CapacityScheduler scheduler = (CapacityScheduler) rm.getResourceScheduler(); // Register 5 nodes, so that we can blacklist atleast one if AM container @@ -122,7 +119,7 @@ public class TestNodeBlacklistingOnAMFailures { // Try the current node a few times for (int i = 0; i <= 2; i++) { currentNode.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); Assert.assertEquals( "AppAttemptState should still be SCHEDULED if currentNode is " @@ -132,7 +129,7 @@ public class TestNodeBlacklistingOnAMFailures { // Now try the other node otherNode.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); // Now the AM container should be allocated MockRM.waitForState(attempt, RMAppAttemptState.ALLOCATED, 20000); @@ -169,8 +166,7 @@ public class TestNodeBlacklistingOnAMFailures { conf.setBoolean(YarnConfiguration.AM_SCHEDULING_NODE_BLACKLISTING_ENABLED, true); - DrainDispatcher dispatcher = new DrainDispatcher(); - MockRM rm = startRM(conf, dispatcher); + MockRM rm = startRM(conf); CapacityScheduler scheduler = (CapacityScheduler) rm.getResourceScheduler(); // Register 5 nodes, so that we can blacklist atleast one if AM container @@ -227,7 +223,7 @@ public class TestNodeBlacklistingOnAMFailures { System.out.println("New AppAttempt launched " + attempt.getAppAttemptId()); nm2.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); // Now the AM container should be allocated MockRM.waitForState(attempt, RMAppAttemptState.ALLOCATED, 20000); @@ -257,8 +253,7 @@ public class TestNodeBlacklistingOnAMFailures { conf.setBoolean(YarnConfiguration.AM_SCHEDULING_NODE_BLACKLISTING_ENABLED, true); - DrainDispatcher dispatcher = new DrainDispatcher(); - MockRM rm = startRM(conf, dispatcher); + MockRM rm = startRM(conf); CapacityScheduler scheduler = (CapacityScheduler) rm.getResourceScheduler(); // Register 5 nodes, so that we can blacklist atleast one if AM container @@ -319,7 +314,7 @@ public class TestNodeBlacklistingOnAMFailures { nm3.nodeHeartbeat(true); nm4.nodeHeartbeat(true); nm5.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); // Now the AM container should be allocated MockRM.waitForState(attempt, RMAppAttemptState.ALLOCATED, 20000); @@ -352,8 +347,7 @@ public class TestNodeBlacklistingOnAMFailures { 1.5f); conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 100); - DrainDispatcher dispatcher = new DrainDispatcher(); - MockRM rm = startRM(conf, dispatcher); + MockRM rm = startRM(conf); MockNM node = new MockNM("127.0.0.1:1234", 8000, rm.getResourceTrackerService()); @@ -367,7 +361,7 @@ public class TestNodeBlacklistingOnAMFailures { // Now the AM container should be allocated RMAppAttempt attempt = MockRM.waitForAttemptScheduled(app, rm); node.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); MockRM.waitForState(attempt, RMAppAttemptState.ALLOCATED, 20000); rm.sendAMLaunched(attempt.getAppAttemptId()); rm.waitForState(attempt.getAppAttemptId(), RMAppAttemptState.LAUNCHED); @@ -394,7 +388,7 @@ public class TestNodeBlacklistingOnAMFailures { .println("New AppAttempt launched " + attempt.getAppAttemptId()); node.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); MockRM.waitForState(attempt, RMAppAttemptState.ALLOCATED, 20000); rm.sendAMLaunched(attempt.getAppAttemptId()); @@ -418,20 +412,13 @@ public class TestNodeBlacklistingOnAMFailures { rm.waitForState(amAttemptID.getApplicationId(), RMAppState.ACCEPTED); } - private MockRM startRM(YarnConfiguration conf, - final DrainDispatcher dispatcher) { - + private MockRM startRM(YarnConfiguration conf) { MemoryRMStateStore memStore = new MemoryRMStateStore(); memStore.init(conf); - MockRM rm1 = new MockRM(conf, memStore) { - @Override - protected Dispatcher createDispatcher() { - return dispatcher; - } - }; + MockRM rm = new MockRM(conf, memStore); - rm1.start(); - return rm1; + rm.start(); + return rm; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestReservationSystemWithRMHA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestReservationSystemWithRMHA.java index 5a6fe67e2b1..f746dc2f188 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestReservationSystemWithRMHA.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestReservationSystemWithRMHA.java @@ -29,7 +29,6 @@ import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.ReservationDefinition; import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.proto.YarnProtos.ReservationAllocationStateProto; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan; @@ -186,9 +185,7 @@ public class TestReservationSystemWithRMHA extends RMHATestBase { rm.registerNode("127.0.0.1:1", memory, vCores); int attempts = 10; do { - DrainDispatcher dispatcher = - (DrainDispatcher) rm1.getRMContext().getDispatcher(); - dispatcher.await(); + rm1.drainEvents(); rm.getRMContext().getReservationSystem() .synchronizePlan(ReservationSystemTestUtil.reservationQ, false); if (rm.getRMContext().getReservationSystem() diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java index c8baa607bf0..f9f0b746233 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java @@ -31,8 +31,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeState; -import org.apache.hadoop.yarn.event.Dispatcher; -import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService; import org.apache.hadoop.yarn.server.resourcemanager.MockAM; @@ -47,12 +45,10 @@ import org.junit.Test; public class TestAMRMRPCNodeUpdates { private MockRM rm; - ApplicationMasterService amService = null; - DrainDispatcher dispatcher = null; + private ApplicationMasterService amService; @Before public void setUp() { - dispatcher = new DrainDispatcher(); this.rm = new MockRM() { @Override public void init(Configuration conf) { @@ -61,12 +57,8 @@ public class TestAMRMRPCNodeUpdates { "1.0"); super.init(conf); } - - @Override - protected Dispatcher createDispatcher() { - return dispatcher; - } }; + rm.start(); amService = rm.getApplicationMasterService(); } @@ -80,14 +72,14 @@ public class TestAMRMRPCNodeUpdates { private void syncNodeHeartbeat(MockNM nm, boolean health) throws Exception { nm.nodeHeartbeat(health); - dispatcher.await(); + rm.drainEvents(); } private void syncNodeLost(MockNM nm) throws Exception { rm.sendNodeStarted(nm); rm.waitForState(nm.getNodeId(), NodeState.RUNNING); rm.sendNodeLost(nm); - dispatcher.await(); + rm.drainEvents(); } private AllocateResponse allocate(final ApplicationAttemptId attemptId, @@ -113,7 +105,7 @@ public class TestAMRMRPCNodeUpdates { MockNM nm2 = rm.registerNode("127.0.0.2:1234", 10000); MockNM nm3 = rm.registerNode("127.0.0.3:1234", 10000); MockNM nm4 = rm.registerNode("127.0.0.4:1234", 10000); - dispatcher.await(); + rm.drainEvents(); RMApp app1 = rm.submitApp(2000); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java index e7c7e51bf2c..6a7325c25c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java @@ -28,7 +28,6 @@ import org.apache.hadoop.yarn.conf.ConfigurationProvider; import org.apache.hadoop.yarn.conf.ConfigurationProviderFactory; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; -import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.event.InlineDispatcher; import org.apache.hadoop.yarn.factories.RecordFactory; @@ -228,21 +227,16 @@ public class TestNMReconnect extends ParameterizedSchedulerTestBase { // The node(127.0.0.1:1234) reconnected with RM. When it registered with // RM, RM set its lastNodeHeartbeatResponse's id to 0 asynchronously. But // the node's heartbeat come before RM succeeded setting the id to 0. - final DrainDispatcher dispatcher = new DrainDispatcher(); - MockRM rm = new MockRM(){ - @Override - protected Dispatcher createDispatcher() { - return dispatcher; - } - }; + MockRM rm = new MockRM(); rm.start(); + MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService()); nm1.registerNode(); int i = 0; while(i < 3) { nm1.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); i++; } @@ -251,7 +245,7 @@ public class TestNMReconnect extends ParameterizedSchedulerTestBase { nm2.registerNode(); RMNode rmNode = rm.getRMContext().getRMNodes().get(nm2.getNodeId()); nm2.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); Assert.assertEquals("Node is Not in Running state.", NodeState.RUNNING, rmNode.getState()); rm.stop(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java index 893f802ade3..db3144898fd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java @@ -46,7 +46,6 @@ import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.server.resourcemanager.MockAM; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java index ff52efd89be..fd17bd91a3d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java @@ -36,8 +36,6 @@ import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.event.Dispatcher; -import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; @@ -612,24 +610,17 @@ public class TestApplicationPriority { conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); conf.setInt(YarnConfiguration.MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY, 10); - final DrainDispatcher dispatcher = new DrainDispatcher(); MemoryRMStateStore memStore = new MemoryRMStateStore(); memStore.init(conf); - MockRM rm1 = new MockRM(conf, memStore) { - @Override - protected Dispatcher createDispatcher() { - return dispatcher; - } - }; + MockRM rm1 = new MockRM(conf, memStore); rm1.start(); MockNM nm1 = new MockNM("127.0.0.1:1234", 16384, rm1.getResourceTrackerService()); nm1.registerNode(); - - dispatcher.await(); + rm1.drainEvents(); ResourceScheduler scheduler = rm1.getRMContext().getScheduler(); LeafQueue defaultQueue = @@ -648,7 +639,7 @@ public class TestApplicationPriority { MockAM am2 = MockRM.launchAM(app2, rm1, nm1); am2.registerAppAttempt(); - dispatcher.await(); + rm1.drainEvents(); Assert.assertEquals(2, defaultQueue.getNumActiveApplications()); Assert.assertEquals(0, defaultQueue.getNumPendingApplications()); @@ -657,7 +648,7 @@ public class TestApplicationPriority { Priority appPriority3 = Priority.newInstance(7); RMApp app3 = rm1.submitApp(memory, appPriority3); - dispatcher.await(); + rm1.drainEvents(); Assert.assertEquals(2, defaultQueue.getNumActiveApplications()); Assert.assertEquals(1, defaultQueue.getNumPendingApplications()); @@ -676,14 +667,8 @@ public class TestApplicationPriority { Assert.assertEquals(app3.getCurrentAppAttempt().getAppAttemptId(), fcApp3.getApplicationAttemptId()); - final DrainDispatcher dispatcher1 = new DrainDispatcher(); // create new RM to represent restart and recover state - MockRM rm2 = new MockRM(conf, memStore) { - @Override - protected Dispatcher createDispatcher() { - return dispatcher1; - } - }; + MockRM rm2 = new MockRM(conf, memStore); // start new RM rm2.start(); @@ -693,7 +678,7 @@ public class TestApplicationPriority { // Verify RM Apps after this restart Assert.assertEquals(3, rm2.getRMContext().getRMApps().size()); - dispatcher1.await(); + rm2.drainEvents(); scheduler = rm2.getRMContext().getScheduler(); defaultQueue = (LeafQueue) ((CapacityScheduler) scheduler).getQueue("default"); @@ -714,7 +699,7 @@ public class TestApplicationPriority { // NM resync to new RM nm1.registerNode(); - dispatcher1.await(); + rm2.drainEvents(); // wait for activating applications count = 50; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java index a4513561d2c..d4e7727ad5e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java @@ -50,8 +50,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.event.Dispatcher; -import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier; import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager; @@ -199,7 +197,6 @@ public class TestClientToAMTokens extends ParameterizedSchedulerTestBase { StartContainersResponse mockResponse = mock(StartContainersResponse.class); when(containerManager.startContainers((StartContainersRequest) any())) .thenReturn(mockResponse); - final DrainDispatcher dispatcher = new DrainDispatcher(); MockRM rm = new MockRMWithCustomAMLauncher(conf, containerManager) { protected ClientRMService createClientRMService() { @@ -208,11 +205,6 @@ public class TestClientToAMTokens extends ParameterizedSchedulerTestBase { getRMContext().getRMDelegationTokenSecretManager()); }; - @Override - protected Dispatcher createDispatcher() { - return dispatcher; - } - @Override protected void doSecureLogin() throws IOException { } @@ -225,11 +217,10 @@ public class TestClientToAMTokens extends ParameterizedSchedulerTestBase { // Set up a node. MockNM nm1 = rm.registerNode("localhost:1234", 3072); nm1.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); - nm1.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); ApplicationAttemptId appAttempt = app.getCurrentAppAttempt().getAppAttemptId(); final MockAM mockAM = @@ -436,7 +427,6 @@ public class TestClientToAMTokens extends ParameterizedSchedulerTestBase { StartContainersResponse mockResponse = mock(StartContainersResponse.class); when(containerManager.startContainers((StartContainersRequest) any())) .thenReturn(mockResponse); - final DrainDispatcher dispatcher = new DrainDispatcher(); MockRM rm = new MockRMWithCustomAMLauncher(conf, containerManager) { protected ClientRMService createClientRMService() { @@ -445,11 +435,6 @@ public class TestClientToAMTokens extends ParameterizedSchedulerTestBase { getRMContext().getRMDelegationTokenSecretManager()); }; - @Override - protected Dispatcher createDispatcher() { - return dispatcher; - } - @Override protected void doSecureLogin() throws IOException { } @@ -462,10 +447,10 @@ public class TestClientToAMTokens extends ParameterizedSchedulerTestBase { // Set up a node. MockNM nm1 = rm.registerNode("localhost:1234", 3072); nm1.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); nm1.nodeHeartbeat(true); - dispatcher.await(); + rm.drainEvents(); ApplicationAttemptId appAttempt = app.getCurrentAppAttempt().getAppAttemptId(); final MockAM mockAM = From 318bfb01bc6793da09e32e9cc292eb63224b6ca2 Mon Sep 17 00:00:00 2001 From: Eric Payne Date: Fri, 31 Mar 2017 12:30:35 -0500 Subject: [PATCH 176/188] YARN-6354. LeveldbRMStateStore can parse invalid keys when recovering reservations. Contributed by Jason Lowe --- .../recovery/LeveldbRMStateStore.java | 8 ++++++++ .../recovery/TestLeveldbRMStateStore.java | 19 +++++++++++++++---- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java index 02f90ddb9af..2ca53db2b34 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/LeveldbRMStateStore.java @@ -214,6 +214,11 @@ public class LeveldbRMStateStore extends RMStateStore { return db == null; } + @VisibleForTesting + DB getDatabase() { + return db; + } + @Override protected Version loadVersion() throws Exception { Version version = null; @@ -284,6 +289,9 @@ public class LeveldbRMStateStore extends RMStateStore { while (iter.hasNext()) { Entry entry = iter.next(); String key = asString(entry.getKey()); + if (!key.startsWith(RM_RESERVATION_KEY_PREFIX)) { + break; + } String planReservationString = key.substring(RM_RESERVATION_KEY_PREFIX.length()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java index 4297e732e77..51adbe189bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestLeveldbRMStateStore.java @@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.records.Version; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; +import org.fusesource.leveldbjni.JniDBFactory; import org.iq80.leveldb.DB; import org.junit.After; import org.junit.Before; @@ -125,17 +126,27 @@ public class TestLeveldbRMStateStore extends RMStateStoreTestBase { public void testCompactionCycle() throws Exception { final DB mockdb = mock(DB.class); conf.setLong(YarnConfiguration.RM_LEVELDB_COMPACTION_INTERVAL_SECS, 1); - LeveldbRMStateStore store = new LeveldbRMStateStore() { + stateStore = new LeveldbRMStateStore() { @Override protected DB openDatabase() throws Exception { return mockdb; } }; - store.init(conf); - store.start(); + stateStore.init(conf); + stateStore.start(); verify(mockdb, timeout(10000)).compactRange( (byte[]) isNull(), (byte[]) isNull()); - store.close(); + } + + @Test + public void testBadKeyIteration() throws Exception { + stateStore = new LeveldbRMStateStore(); + stateStore.init(conf); + stateStore.start(); + DB db = stateStore.getDatabase(); + // add an entry that appears at the end of the database when iterating + db.put(JniDBFactory.bytes("zzz"), JniDBFactory.bytes("z")); + stateStore.loadState(); } class LeveldbStateStoreTester implements RMStateStoreHelper { From 5485d93bda3329a7c80767c3723cc6e1a9233dbc Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Fri, 31 Mar 2017 12:10:20 -0700 Subject: [PATCH 177/188] HDFS-11603. Improve slow mirror/disk warnings in BlockReceiver. --- .../hadoop/hdfs/protocol/DatanodeInfo.java | 1 + .../hdfs/server/datanode/BlockReceiver.java | 61 ++++++++++++++----- 2 files changed, 47 insertions(+), 15 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java index e1698c98c03..0a8c9151f1c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java @@ -55,6 +55,7 @@ public class DatanodeInfo extends DatanodeID implements Node { private String softwareVersion; private List dependentHostNames = new LinkedList<>(); private String upgradeDomain; + public static final DatanodeInfo[] EMPTY_ARRAY = {}; // Datanode administrative states public enum AdminStates { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java index bb6bd5570e2..00109e052d9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java @@ -100,6 +100,7 @@ class BlockReceiver implements Closeable { private DataTransferThrottler throttler; private ReplicaOutputStreams streams; private DatanodeInfo srcDataNode = null; + private DatanodeInfo[] downstreamDNs = DatanodeInfo.EMPTY_ARRAY; private final DataNode datanode; volatile private boolean mirrorError; @@ -424,10 +425,10 @@ class BlockReceiver implements Closeable { } } long duration = Time.monotonicNow() - begin; - if (duration > datanodeSlowLogThresholdMs) { + if (duration > datanodeSlowLogThresholdMs && LOG.isWarnEnabled()) { LOG.warn("Slow flushOrSync took " + duration + "ms (threshold=" + datanodeSlowLogThresholdMs + "ms), isSync:" + isSync + ", flushTotalNanos=" - + flushTotalNanos + "ns"); + + flushTotalNanos + "ns, volume=" + getVolumeBaseUri()); } } @@ -578,9 +579,10 @@ class BlockReceiver implements Closeable { mirrorAddr, duration); trackSendPacketToLastNodeInPipeline(duration); - if (duration > datanodeSlowLogThresholdMs) { + if (duration > datanodeSlowLogThresholdMs && LOG.isWarnEnabled()) { LOG.warn("Slow BlockReceiver write packet to mirror took " + duration - + "ms (threshold=" + datanodeSlowLogThresholdMs + "ms)"); + + "ms (threshold=" + datanodeSlowLogThresholdMs + "ms), " + + "downstream DNs=" + Arrays.toString(downstreamDNs)); } } catch (IOException e) { handleMirrorOutError(e); @@ -711,9 +713,10 @@ class BlockReceiver implements Closeable { streams.writeDataToDisk(dataBuf.array(), startByteToDisk, numBytesToDisk); long duration = Time.monotonicNow() - begin; - if (duration > datanodeSlowLogThresholdMs) { + if (duration > datanodeSlowLogThresholdMs && LOG.isWarnEnabled()) { LOG.warn("Slow BlockReceiver write data to disk cost:" + duration - + "ms (threshold=" + datanodeSlowLogThresholdMs + "ms)"); + + "ms (threshold=" + datanodeSlowLogThresholdMs + "ms), " + + "volume=" + getVolumeBaseUri()); } if (duration > maxWriteToDiskMs) { @@ -902,9 +905,10 @@ class BlockReceiver implements Closeable { } lastCacheManagementOffset = offsetInBlock; long duration = Time.monotonicNow() - begin; - if (duration > datanodeSlowLogThresholdMs) { + if (duration > datanodeSlowLogThresholdMs && LOG.isWarnEnabled()) { LOG.warn("Slow manageWriterOsCache took " + duration - + "ms (threshold=" + datanodeSlowLogThresholdMs + "ms)"); + + "ms (threshold=" + datanodeSlowLogThresholdMs + + "ms), volume=" + getVolumeBaseUri()); } } } catch (Throwable t) { @@ -932,13 +936,7 @@ class BlockReceiver implements Closeable { boolean responderClosed = false; mirrorOut = mirrOut; mirrorAddr = mirrAddr; - isPenultimateNode = ((downstreams != null) && (downstreams.length == 1)); - if (isPenultimateNode) { - mirrorNameForMetrics = (downstreams[0].getInfoSecurePort() != 0 ? - downstreams[0].getInfoSecureAddr() : downstreams[0].getInfoAddr()); - LOG.debug("Will collect peer metrics for downstream node {}", - mirrorNameForMetrics); - } + initPerfMonitoring(downstreams); throttler = throttlerArg; this.replyOut = replyOut; @@ -1058,6 +1056,39 @@ class BlockReceiver implements Closeable { } } + /** + * If we have downstream DNs and peerMetrics are enabled, then initialize + * some state for monitoring the performance of downstream DNs. + * + * @param downstreams downstream DNs, or null if there are none. + */ + private void initPerfMonitoring(DatanodeInfo[] downstreams) { + if (downstreams != null && downstreams.length > 0) { + downstreamDNs = downstreams; + isPenultimateNode = (downstreams.length == 1); + if (isPenultimateNode && datanode.getPeerMetrics() != null) { + mirrorNameForMetrics = (downstreams[0].getInfoSecurePort() != 0 ? + downstreams[0].getInfoSecureAddr() : downstreams[0].getInfoAddr()); + LOG.debug("Will collect peer metrics for downstream node {}", + mirrorNameForMetrics); + } + } + } + + /** + * Fetch the base URI of the volume on which this replica resides. + * + * @returns Volume base URI as string if available. Else returns the + * the string "unavailable". + */ + private String getVolumeBaseUri() { + final ReplicaInfo ri = replicaInfo.getReplicaInfo(); + if (ri != null && ri.getVolume() != null) { + return ri.getVolume().getBaseURI().toString(); + } + return "unavailable"; + } + /** Cleanup a partial block * if this write is for a replication request (and not from a client) */ From 73835c73e2d34b3854a71dd29d88c8303d698ac8 Mon Sep 17 00:00:00 2001 From: Hanisha Koneru Date: Fri, 31 Mar 2017 13:50:29 -0700 Subject: [PATCH 178/188] HDFS-11560. Expose slow disks via NameNode JMX. Contributed by Hanisha Koneru. --- .../blockmanagement/DatanodeManager.java | 9 +++ .../blockmanagement/SlowDiskTracker.java | 3 + .../hadoop/hdfs/server/namenode/NameNode.java | 6 ++ .../server/namenode/NameNodeStatusMXBean.java | 8 +++ .../blockmanagement/TestSlowDiskTracker.java | 13 +--- .../namenode/TestNameNodeStatusMXBean.java | 59 ++++++++++++++++++- 6 files changed, 85 insertions(+), 13 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java index 18135a8a549..c7bdca9c155 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java @@ -1907,5 +1907,14 @@ public class DatanodeManager { public SlowDiskTracker getSlowDiskTracker() { return slowDiskTracker; } + /** + * Retrieve information about slow disks as a JSON. + * Returns null if we are not tracking slow disks. + * @return + */ + public String getSlowDisksReport() { + return slowDiskTracker != null ? + slowDiskTracker.getSlowDiskReportAsJsonString() : null; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowDiskTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowDiskTracker.java index 25920a2e07f..52fce5d35c0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowDiskTracker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SlowDiskTracker.java @@ -256,6 +256,9 @@ public class SlowDiskTracker { public String getSlowDiskReportAsJsonString() { ObjectMapper objectMapper = new ObjectMapper(); try { + if (slowDisksReport.isEmpty()) { + return null; + } return objectMapper.writeValueAsString(slowDisksReport); } catch (JsonProcessingException e) { // Failed to serialize. Don't log the exception call stack. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index e7841f02a9e..32d268a9dbf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -1826,6 +1826,12 @@ public class NameNode extends ReconfigurableBase implements .getSlowPeersReport(); } + @Override //NameNodeStatusMXBean + public String getSlowDisksReport() { + return namesystem.getBlockManager().getDatanodeManager() + .getSlowDisksReport(); + } + /** * Shutdown the NN immediately in an ungraceful way. Used when it would be * unsafe for the NN to continue operating, e.g. during a failed HA state diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeStatusMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeStatusMXBean.java index f46b9ae927e..ed1b96bdf11 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeStatusMXBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeStatusMXBean.java @@ -75,4 +75,12 @@ public interface NameNodeStatusMXBean { * enabled. The report is in a JSON format. */ String getSlowPeersReport(); + + + /** + * Gets the topN slow disks in the cluster, if the feature is enabled. + * + * @return JSON string of list of diskIDs and latencies + */ + String getSlowDisksReport(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSlowDiskTracker.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSlowDiskTracker.java index e96b96a25af..16dfab208ad 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSlowDiskTracker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSlowDiskTracker.java @@ -393,18 +393,9 @@ public class TestSlowDiskTracker { timer.advance(reportValidityMs); tracker.updateSlowDiskReportAsync(timer.monotonicNow()); + Thread.sleep(OUTLIERS_REPORT_INTERVAL*2); - GenericTestUtils.waitFor(new Supplier() { - @Override - public Boolean get() { - return tracker.getSlowDiskReportAsJsonString() != null; - } - }, 500, 5000); - - ArrayList jsonReport = getAndDeserializeJson( - tracker.getSlowDiskReportAsJsonString()); - - assertTrue(jsonReport.isEmpty()); + assertTrue(tracker.getSlowDiskReportAsJsonString() == null); } private boolean isDiskInReports(ArrayList reports, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeStatusMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeStatusMXBean.java index c03dc209f9d..8fe734e44bc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeStatusMXBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeStatusMXBean.java @@ -17,17 +17,23 @@ */ package org.apache.hadoop.hdfs.server.namenode; +import com.google.common.base.Supplier; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager; +import org.apache.hadoop.hdfs.server.datanode.DataNode; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.server.datanode.TestDataNodeMXBean; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.Assert; import org.junit.Test; import javax.management.MBeanServer; import javax.management.ObjectName; import java.lang.management.ManagementFactory; +import java.util.List; +import java.util.concurrent.TimeUnit; /** * Class for testing {@link NameNodeStatusMXBean} implementation. @@ -38,7 +44,7 @@ public class TestNameNodeStatusMXBean { TestNameNodeStatusMXBean.class); @Test(timeout = 120000L) - public void testDataNodeMXBean() throws Exception { + public void testNameNodeStatusMXBean() throws Exception { Configuration conf = new Configuration(); MiniDFSCluster cluster = null; @@ -84,6 +90,55 @@ public class TestNameNodeStatusMXBean { String slowPeersReport = (String)mbs.getAttribute(mxbeanName, "SlowPeersReport"); Assert.assertEquals(nn.getSlowPeersReport(), slowPeersReport); + + // Get attribute "SlowDisksReport" + String slowDisksReport = (String)mbs.getAttribute(mxbeanName, + "SlowDisksReport"); + Assert.assertEquals(nn.getSlowDisksReport(), slowDisksReport); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } + + @Test + public void testNameNodeMXBeanSlowDisksEnabled() throws Exception { + Configuration conf = new Configuration(); + conf.setDouble( + DFSConfigKeys.DFS_DATANODE_FILEIO_PROFILING_SAMPLING_FRACTION_KEY, 1.0); + conf.setTimeDuration( + DFSConfigKeys.DFS_DATANODE_OUTLIERS_REPORT_INTERVAL_KEY, + 1000, TimeUnit.MILLISECONDS); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); + + try { + List datanodes = cluster.getDataNodes(); + Assert.assertEquals(datanodes.size(), 1); + DataNode datanode = datanodes.get(0); + String slowDiskPath = "test/data1/slowVolume"; + datanode.getDiskMetrics().addSlowDiskForTesting(slowDiskPath, null); + + NameNode nn = cluster.getNameNode(); + DatanodeManager datanodeManager = nn.getNamesystem().getBlockManager() + .getDatanodeManager(); + + MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); + ObjectName mxbeanName = new ObjectName( + "Hadoop:service=NameNode,name=NameNodeStatus"); + + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + return (datanodeManager.getSlowDisksReport() != null); + } + }, 1000, 100000); + + String slowDisksReport = (String)mbs.getAttribute( + mxbeanName, "SlowDisksReport"); + Assert.assertEquals(datanodeManager.getSlowDisksReport(), + slowDisksReport); + Assert.assertTrue(slowDisksReport.contains(slowDiskPath)); } finally { if (cluster != null) { cluster.shutdown(); From 26172a94d6431e70d7fe15d66be9a7e195f79f60 Mon Sep 17 00:00:00 2001 From: Mingliang Liu Date: Thu, 23 Jun 2016 00:21:49 -0700 Subject: [PATCH 179/188] HADOOP-14267. Make DistCpOptions immutable. Contributed by Mingliang Liu --- .../org/apache/hadoop/tools/CopyListing.java | 38 +- .../java/org/apache/hadoop/tools/DistCp.java | 106 +- .../apache/hadoop/tools/DistCpContext.java | 198 ++++ .../hadoop/tools/DistCpOptionSwitch.java | 2 +- .../apache/hadoop/tools/DistCpOptions.java | 925 ++++++++---------- .../org/apache/hadoop/tools/DistCpSync.java | 42 +- .../hadoop/tools/FileBasedCopyListing.java | 12 +- .../hadoop/tools/GlobbedCopyListing.java | 17 +- .../apache/hadoop/tools/OptionsParser.java | 307 ++---- .../hadoop/tools/SimpleCopyListing.java | 115 ++- .../hadoop/tools/mapred/CopyCommitter.java | 15 +- .../apache/hadoop/tools/util/DistCpUtils.java | 8 +- .../apache/hadoop/tools/TestCopyListing.java | 51 +- .../hadoop/tools/TestDistCpOptions.java | 500 ++++++++++ .../apache/hadoop/tools/TestDistCpSync.java | 68 +- .../tools/TestDistCpSyncReverseBase.java | 44 +- .../apache/hadoop/tools/TestDistCpViewFs.java | 10 +- .../tools/TestFileBasedCopyListing.java | 9 +- .../hadoop/tools/TestGlobbedCopyListing.java | 11 +- .../apache/hadoop/tools/TestIntegration.java | 20 +- .../hadoop/tools/TestOptionsParser.java | 81 +- .../contract/AbstractContractDistCpTest.java | 6 +- .../tools/mapred/TestCopyCommitter.java | 34 +- .../mapred/TestUniformSizeInputFormat.java | 15 +- .../mapred/lib/TestDynamicInputFormat.java | 17 +- 25 files changed, 1578 insertions(+), 1073 deletions(-) create mode 100644 hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpContext.java create mode 100644 hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpOptions.java diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java index 9ebf9d29b3b..908b5580476 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java @@ -77,41 +77,41 @@ public abstract class CopyListing extends Configured { * TARGET IS DIR : Key-"/file1", Value-FileStatus(/tmp/file1) * * @param pathToListFile - Output file where the listing would be stored - * @param options - Input options to distcp + * @param distCpContext - distcp context associated with input options * @throws IOException - Exception if any */ public final void buildListing(Path pathToListFile, - DistCpOptions options) throws IOException { - validatePaths(options); - doBuildListing(pathToListFile, options); + DistCpContext distCpContext) throws IOException { + validatePaths(distCpContext); + doBuildListing(pathToListFile, distCpContext); Configuration config = getConf(); config.set(DistCpConstants.CONF_LABEL_LISTING_FILE_PATH, pathToListFile.toString()); config.setLong(DistCpConstants.CONF_LABEL_TOTAL_BYTES_TO_BE_COPIED, getBytesToCopy()); config.setLong(DistCpConstants.CONF_LABEL_TOTAL_NUMBER_OF_RECORDS, getNumberOfPaths()); - validateFinalListing(pathToListFile, options); + validateFinalListing(pathToListFile, distCpContext); LOG.info("Number of paths in the copy list: " + this.getNumberOfPaths()); } /** * Validate input and output paths * - * @param options - Input options + * @param distCpContext - Distcp context * @throws InvalidInputException If inputs are invalid * @throws IOException any Exception with FS */ - protected abstract void validatePaths(DistCpOptions options) + protected abstract void validatePaths(DistCpContext distCpContext) throws IOException, InvalidInputException; /** * The interface to be implemented by sub-classes, to create the source/target file listing. * @param pathToListFile Path on HDFS where the listing file is written. - * @param options Input Options for DistCp (indicating source/target paths.) + * @param distCpContext - Distcp context * @throws IOException Thrown on failure to create the listing file. */ protected abstract void doBuildListing(Path pathToListFile, - DistCpOptions options) throws IOException; + DistCpContext distCpContext) throws IOException; /** * Return the total bytes that distCp should copy for the source paths @@ -135,17 +135,17 @@ public abstract class CopyListing extends Configured { * If preserving XAttrs, checks that file system can support XAttrs. * * @param pathToListFile - path listing build by doBuildListing - * @param options - Input options to distcp + * @param context - Distcp context with associated input options * @throws IOException - Any issues while checking for duplicates and throws * @throws DuplicateFileException - if there are duplicates */ - private void validateFinalListing(Path pathToListFile, DistCpOptions options) + private void validateFinalListing(Path pathToListFile, DistCpContext context) throws DuplicateFileException, IOException { Configuration config = getConf(); FileSystem fs = pathToListFile.getFileSystem(config); - final boolean splitLargeFile = options.splitLargeFile(); + final boolean splitLargeFile = context.splitLargeFile(); // When splitLargeFile is enabled, we don't randomize the copylist // earlier, so we don't do the sorting here. For a file that has @@ -188,7 +188,7 @@ public abstract class CopyListing extends Configured { } } reader.getCurrentValue(lastFileStatus); - if (options.shouldPreserve(DistCpOptions.FileAttribute.ACL)) { + if (context.shouldPreserve(DistCpOptions.FileAttribute.ACL)) { FileSystem lastFs = lastFileStatus.getPath().getFileSystem(config); URI lastFsUri = lastFs.getUri(); if (!aclSupportCheckFsSet.contains(lastFsUri)) { @@ -196,7 +196,7 @@ public abstract class CopyListing extends Configured { aclSupportCheckFsSet.add(lastFsUri); } } - if (options.shouldPreserve(DistCpOptions.FileAttribute.XATTR)) { + if (context.shouldPreserve(DistCpOptions.FileAttribute.XATTR)) { FileSystem lastFs = lastFileStatus.getPath().getFileSystem(config); URI lastFsUri = lastFs.getUri(); if (!xAttrSupportCheckFsSet.contains(lastFsUri)) { @@ -210,7 +210,7 @@ public abstract class CopyListing extends Configured { lastChunkOffset = lastFileStatus.getChunkOffset(); lastChunkLength = lastFileStatus.getChunkLength(); } - if (options.shouldUseDiff() && LOG.isDebugEnabled()) { + if (context.shouldUseDiff() && LOG.isDebugEnabled()) { LOG.debug("Copy list entry " + idx + ": " + lastFileStatus.getPath().toUri().getPath()); idx++; @@ -253,14 +253,12 @@ public abstract class CopyListing extends Configured { * Public Factory method with which the appropriate CopyListing implementation may be retrieved. * @param configuration The input configuration. * @param credentials Credentials object on which the FS delegation tokens are cached - * @param options The input Options, to help choose the appropriate CopyListing Implementation. + * @param context Distcp context with associated input options * @return An instance of the appropriate CopyListing implementation. * @throws java.io.IOException - Exception if any */ public static CopyListing getCopyListing(Configuration configuration, - Credentials credentials, - DistCpOptions options) - throws IOException { + Credentials credentials, DistCpContext context) throws IOException { String copyListingClassName = configuration.get(DistCpConstants. CONF_LABEL_COPY_LISTING_CLASS, ""); Class copyListingClass; @@ -270,7 +268,7 @@ public abstract class CopyListing extends Configured { CONF_LABEL_COPY_LISTING_CLASS, GlobbedCopyListing.class, CopyListing.class); } else { - if (options.getSourceFileListing() == null) { + if (context.getSourceFileListing() == null) { copyListingClass = GlobbedCopyListing.class; } else { copyListingClass = FileBasedCopyListing.class; diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java index 8c2fa24a15c..df9c32896f3 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java @@ -21,6 +21,7 @@ package org.apache.hadoop.tools; import java.io.IOException; import java.util.Random; +import com.google.common.base.Preconditions; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -35,7 +36,6 @@ import org.apache.hadoop.mapreduce.Cluster; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.JobSubmissionFiles; -import org.apache.hadoop.tools.DistCpOptions.FileAttribute; import org.apache.hadoop.tools.CopyListing.*; import org.apache.hadoop.tools.mapred.CopyMapper; import org.apache.hadoop.tools.mapred.CopyOutputFormat; @@ -66,7 +66,9 @@ public class DistCp extends Configured implements Tool { static final Log LOG = LogFactory.getLog(DistCp.class); - private DistCpOptions inputOptions; + @VisibleForTesting + DistCpContext context; + private Path metaFolder; private static final String PREFIX = "_distcp"; @@ -79,15 +81,14 @@ public class DistCp extends Configured implements Tool { private FileSystem jobFS; private void prepareFileListing(Job job) throws Exception { - if (inputOptions.shouldUseSnapshotDiff()) { + if (context.shouldUseSnapshotDiff()) { // When "-diff" or "-rdiff" is passed, do sync() first, then // create copyListing based on snapshot diff. - DistCpSync distCpSync = new DistCpSync(inputOptions, getConf()); + DistCpSync distCpSync = new DistCpSync(context, getConf()); if (distCpSync.sync()) { createInputFileListingWithDiff(job, distCpSync); } else { - throw new Exception("DistCp sync failed, input options: " - + inputOptions); + throw new Exception("DistCp sync failed, input options: " + context); } } else { // When no "-diff" or "-rdiff" is passed, create copyListing @@ -99,16 +100,19 @@ public class DistCp extends Configured implements Tool { /** * Public Constructor. Creates DistCp object with specified input-parameters. * (E.g. source-paths, target-location, etc.) - * @param inputOptions Options (indicating source-paths, target-location.) - * @param configuration The Hadoop configuration against which the Copy-mapper must run. + * @param configuration configuration against which the Copy-mapper must run + * @param inputOptions Immutable options * @throws Exception */ - public DistCp(Configuration configuration, DistCpOptions inputOptions) throws Exception { + public DistCp(Configuration configuration, DistCpOptions inputOptions) + throws Exception { Configuration config = new Configuration(configuration); config.addResource(DISTCP_DEFAULT_XML); config.addResource(DISTCP_SITE_XML); setConf(config); - this.inputOptions = inputOptions; + if (inputOptions != null) { + this.context = new DistCpContext(inputOptions); + } this.metaFolder = createMetaFolderPath(); } @@ -134,10 +138,10 @@ public class DistCp extends Configured implements Tool { } try { - inputOptions = (OptionsParser.parse(argv)); - setOptionsForSplitLargeFile(); + context = new DistCpContext(OptionsParser.parse(argv)); + checkSplitLargeFile(); setTargetPathExists(); - LOG.info("Input Options: " + inputOptions); + LOG.info("Input Options: " + context); } catch (Throwable e) { LOG.error("Invalid arguments: ", e); System.err.println("Invalid arguments: " + e.getMessage()); @@ -173,9 +177,11 @@ public class DistCp extends Configured implements Tool { * @throws Exception */ public Job execute() throws Exception { + Preconditions.checkState(context != null, + "The DistCpContext should have been created before running DistCp!"); Job job = createAndSubmitJob(); - if (inputOptions.shouldBlock()) { + if (context.shouldBlock()) { waitForJobCompletion(job); } return job; @@ -186,7 +192,7 @@ public class DistCp extends Configured implements Tool { * @return The mapreduce job object that has been submitted */ public Job createAndSubmitJob() throws Exception { - assert inputOptions != null; + assert context != null; assert getConf() != null; Job job = null; try { @@ -230,53 +236,36 @@ public class DistCp extends Configured implements Tool { * for the benefit of CopyCommitter */ private void setTargetPathExists() throws IOException { - Path target = inputOptions.getTargetPath(); + Path target = context.getTargetPath(); FileSystem targetFS = target.getFileSystem(getConf()); boolean targetExists = targetFS.exists(target); - inputOptions.setTargetPathExists(targetExists); + context.setTargetPathExists(targetExists); getConf().setBoolean(DistCpConstants.CONF_LABEL_TARGET_PATH_EXISTS, targetExists); } /** - * Check if concat is supported by fs. - * Throws UnsupportedOperationException if not. + * Check splitting large files is supported and populate configs. */ - private void checkConcatSupport(FileSystem fs) { + private void checkSplitLargeFile() throws IOException { + if (!context.splitLargeFile()) { + return; + } + + final Path target = context.getTargetPath(); + final FileSystem targetFS = target.getFileSystem(getConf()); try { Path[] src = null; Path tgt = null; - fs.concat(tgt, src); + targetFS.concat(tgt, src); } catch (UnsupportedOperationException use) { throw new UnsupportedOperationException( DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch() + - " is not supported since the target file system doesn't" + - " support concat.", use); + " is not supported since the target file system doesn't" + + " support concat.", use); } catch (Exception e) { // Ignore other exception } - } - - /** - * Set up needed options for splitting large files. - */ - private void setOptionsForSplitLargeFile() throws IOException { - if (!inputOptions.splitLargeFile()) { - return; - } - Path target = inputOptions.getTargetPath(); - FileSystem targetFS = target.getFileSystem(getConf()); - checkConcatSupport(targetFS); - - LOG.info("Enabling preserving blocksize since " - + DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch() + " is passed."); - inputOptions.preserve(FileAttribute.BLOCKSIZE); - - LOG.info("Set " + - DistCpOptionSwitch.APPEND.getSwitch() - + " to false since " + DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch() - + " is passed."); - inputOptions.setAppend(false); LOG.info("Set " + DistCpConstants.CONF_LABEL_SIMPLE_LISTING_RANDOMIZE_FILES @@ -286,7 +275,6 @@ public class DistCp extends Configured implements Tool { DistCpConstants.CONF_LABEL_SIMPLE_LISTING_RANDOMIZE_FILES, false); } - /** * Create Job object for submitting it, with all the configuration * @@ -300,7 +288,7 @@ public class DistCp extends Configured implements Tool { jobName += ": " + userChosenName; Job job = Job.getInstance(getConf()); job.setJobName(jobName); - job.setInputFormatClass(DistCpUtils.getStrategy(getConf(), inputOptions)); + job.setInputFormatClass(DistCpUtils.getStrategy(getConf(), context)); job.setJarByClass(CopyMapper.class); configureOutputFormat(job); @@ -311,9 +299,9 @@ public class DistCp extends Configured implements Tool { job.setOutputFormatClass(CopyOutputFormat.class); job.getConfiguration().set(JobContext.MAP_SPECULATIVE, "false"); job.getConfiguration().set(JobContext.NUM_MAPS, - String.valueOf(inputOptions.getMaxMaps())); + String.valueOf(context.getMaxMaps())); - inputOptions.appendToConf(job.getConfiguration()); + context.appendToConf(job.getConfiguration()); return job; } @@ -325,18 +313,20 @@ public class DistCp extends Configured implements Tool { */ private void configureOutputFormat(Job job) throws IOException { final Configuration configuration = job.getConfiguration(); - Path targetPath = inputOptions.getTargetPath(); + Path targetPath = context.getTargetPath(); FileSystem targetFS = targetPath.getFileSystem(configuration); targetPath = targetPath.makeQualified(targetFS.getUri(), targetFS.getWorkingDirectory()); - if (inputOptions.shouldPreserve(DistCpOptions.FileAttribute.ACL)) { + if (context.shouldPreserve( + DistCpOptions.FileAttribute.ACL)) { DistCpUtils.checkFileSystemAclSupport(targetFS); } - if (inputOptions.shouldPreserve(DistCpOptions.FileAttribute.XATTR)) { + if (context.shouldPreserve( + DistCpOptions.FileAttribute.XATTR)) { DistCpUtils.checkFileSystemXAttrSupport(targetFS); } - if (inputOptions.shouldAtomicCommit()) { - Path workDir = inputOptions.getAtomicWorkPath(); + if (context.shouldAtomicCommit()) { + Path workDir = context.getAtomicWorkPath(); if (workDir == null) { workDir = targetPath.getParent(); } @@ -353,7 +343,7 @@ public class DistCp extends Configured implements Tool { } CopyOutputFormat.setCommitDirectory(job, targetPath); - Path logPath = inputOptions.getLogPath(); + Path logPath = context.getLogPath(); if (logPath == null) { logPath = new Path(metaFolder, "_logs"); } else { @@ -374,8 +364,8 @@ public class DistCp extends Configured implements Tool { protected Path createInputFileListing(Job job) throws IOException { Path fileListingPath = getFileListingPath(); CopyListing copyListing = CopyListing.getCopyListing(job.getConfiguration(), - job.getCredentials(), inputOptions); - copyListing.buildListing(fileListingPath, inputOptions); + job.getCredentials(), context); + copyListing.buildListing(fileListingPath, context); return fileListingPath; } @@ -391,7 +381,7 @@ public class DistCp extends Configured implements Tool { Path fileListingPath = getFileListingPath(); CopyListing copyListing = new SimpleCopyListing(job.getConfiguration(), job.getCredentials(), distCpSync); - copyListing.buildListing(fileListingPath, inputOptions); + copyListing.buildListing(fileListingPath, context); return fileListingPath; } diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpContext.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpContext.java new file mode 100644 index 00000000000..c34005e6c4b --- /dev/null +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpContext.java @@ -0,0 +1,198 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.tools; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.tools.DistCpOptions.FileAttribute; + +import java.util.List; +import java.util.Set; + +/** + * This is the context of the distcp at runtime. + * + * It has the immutable {@link DistCpOptions} and mutable runtime status. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class DistCpContext { + private final DistCpOptions options; + + /** The source paths can be set at runtime via snapshots. */ + private List sourcePaths; + + /** This is a derived field, it's initialized in the beginning of distcp. */ + private boolean targetPathExists = true; + + /** Indicate that raw.* xattrs should be preserved if true. */ + private boolean preserveRawXattrs = false; + + public DistCpContext(DistCpOptions options) { + this.options = options; + this.sourcePaths = options.getSourcePaths(); + } + + public void setSourcePaths(List sourcePaths) { + this.sourcePaths = sourcePaths; + } + + /** + * @return the sourcePaths. Please note this method does not directly delegate + * to the {@link #options}. + */ + public List getSourcePaths() { + return sourcePaths; + } + + public Path getSourceFileListing() { + return options.getSourceFileListing(); + } + + public Path getTargetPath() { + return options.getTargetPath(); + } + + public boolean shouldAtomicCommit() { + return options.shouldAtomicCommit(); + } + + public boolean shouldSyncFolder() { + return options.shouldSyncFolder(); + } + + public boolean shouldDeleteMissing() { + return options.shouldDeleteMissing(); + } + + public boolean shouldIgnoreFailures() { + return options.shouldIgnoreFailures(); + } + + public boolean shouldOverwrite() { + return options.shouldOverwrite(); + } + + public boolean shouldAppend() { + return options.shouldAppend(); + } + + public boolean shouldSkipCRC() { + return options.shouldSkipCRC(); + } + + public boolean shouldBlock() { + return options.shouldBlock(); + } + + public boolean shouldUseDiff() { + return options.shouldUseDiff(); + } + + public boolean shouldUseRdiff() { + return options.shouldUseRdiff(); + } + + public boolean shouldUseSnapshotDiff() { + return options.shouldUseSnapshotDiff(); + } + + public String getFromSnapshot() { + return options.getFromSnapshot(); + } + + public String getToSnapshot() { + return options.getToSnapshot(); + } + + public final String getFiltersFile() { + return options.getFiltersFile(); + } + + public int getNumListstatusThreads() { + return options.getNumListstatusThreads(); + } + + public int getMaxMaps() { + return options.getMaxMaps(); + } + + public float getMapBandwidth() { + return options.getMapBandwidth(); + } + + public Set getPreserveAttributes() { + return options.getPreserveAttributes(); + } + + public boolean shouldPreserve(FileAttribute attribute) { + return options.shouldPreserve(attribute); + } + + public boolean shouldPreserveRawXattrs() { + return preserveRawXattrs; + } + + public void setPreserveRawXattrs(boolean preserveRawXattrs) { + this.preserveRawXattrs = preserveRawXattrs; + } + + public Path getAtomicWorkPath() { + return options.getAtomicWorkPath(); + } + + public Path getLogPath() { + return options.getLogPath(); + } + + public String getCopyStrategy() { + return options.getCopyStrategy(); + } + + public int getBlocksPerChunk() { + return options.getBlocksPerChunk(); + } + + public final boolean splitLargeFile() { + return options.getBlocksPerChunk() > 0; + } + + public void setTargetPathExists(boolean targetPathExists) { + this.targetPathExists = targetPathExists; + } + + public boolean isTargetPathExists() { + return targetPathExists; + } + + public void appendToConf(Configuration conf) { + options.appendToConf(conf); + } + + @Override + public String toString() { + return options.toString() + + ", sourcePaths=" + sourcePaths + + ", targetPathExists=" + targetPathExists + + ", preserveRawXattrs" + preserveRawXattrs; + } + +} diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java index ced9b540c89..81abb7df13d 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java @@ -81,7 +81,7 @@ public enum DistCpOptionSwitch { NUM_LISTSTATUS_THREADS(DistCpConstants.CONF_LABEL_LISTSTATUS_THREADS, new Option("numListstatusThreads", true, "Number of threads to " + "use for building file listing (max " + - DistCpOptions.maxNumListstatusThreads + ").")), + DistCpOptions.MAX_NUM_LISTSTATUS_THREADS + ").")), /** * Max number of maps to use during copy. DistCp will split work * as equally as possible among these maps diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java index 9822d83dbd0..97ae0c4ef1a 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java @@ -18,43 +18,88 @@ package org.apache.hadoop.tools; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.tools.util.DistCpUtils; +import java.util.Collections; import java.util.EnumSet; -import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; +import java.util.Set; /** * The Options class encapsulates all DistCp options. - * These may be set from command-line (via the OptionsParser) - * or may be set manually. + * + * When you add a new option, please: + * - Add the field along with javadoc in DistCpOptions and its Builder + * - Add setter method in the {@link Builder} class + * + * This class is immutable. */ -public class DistCpOptions { +public final class DistCpOptions { + private static final Logger LOG = LoggerFactory.getLogger(Builder.class); + public static final int MAX_NUM_LISTSTATUS_THREADS = 40; + + /** File path (hdfs:// or file://) that contains the list of actual files to + * copy. + */ + private final Path sourceFileListing; + + /** List of source-paths (including wildcards) to be copied to target. */ + private final List sourcePaths; + + /** Destination path for the dist-copy. */ + private final Path targetPath; + + /** Whether data need to be committed automatically. */ + private final boolean atomicCommit; + + /** the work path for atomic commit. If null, the work + * path would be parentOf(targetPath) + "/._WIP_" + nameOf(targetPath). */ + private final Path atomicWorkPath; + + /** Whether source and target folder contents be sync'ed up. */ + private final boolean syncFolder; + + /** Whether files only present in target should be deleted. */ + private boolean deleteMissing; + + /** Whether failures during copy be ignored. */ + private final boolean ignoreFailures; + + /** Whether files should always be overwritten on target. */ + private final boolean overwrite; + + /** Whether we want to append new data to target files. This is valid only + * with update option and CRC is not skipped. */ + private final boolean append; + + /** Whether checksum comparison should be skipped while determining if source + * and destination files are identical. */ + private final boolean skipCRC; + + /** Whether to run blocking or non-blocking. */ + private final boolean blocking; - private boolean atomicCommit = false; - private boolean syncFolder = false; - private boolean deleteMissing = false; - private boolean ignoreFailures = false; - private boolean overwrite = false; - private boolean append = false; - private boolean skipCRC = false; - private boolean blocking = true; // When "-diff s1 s2 src tgt" is passed, apply forward snapshot diff (from s1 // to s2) of source cluster to the target cluster to sync target cluster with // the source cluster. Referred to as "Fdiff" in the code. // It's required that s2 is newer than s1. - private boolean useDiff = false; + private final boolean useDiff; // When "-rdiff s2 s1 src tgt" is passed, apply reversed snapshot diff (from // s2 to s1) of target cluster to the target cluster, so to make target // cluster go back to s1. Referred to as "Rdiff" in the code. // It's required that s2 is newer than s1, and src and tgt have exact same // content at their s1, if src is not the same as tgt. - private boolean useRdiff = false; + private final boolean useRdiff; // For both -diff and -rdiff, given the example command line switches, two // steps are taken: @@ -66,44 +111,53 @@ public class DistCpOptions { // could be the tgt itself (HDFS-9820). // - public static final int maxNumListstatusThreads = 40; - private int numListstatusThreads = 0; // Indicates that flag is not set. - private int maxMaps = DistCpConstants.DEFAULT_MAPS; - private float mapBandwidth = 0; // Indicates that we should use the default. + private final String fromSnapshot; + private final String toSnapshot; - private String copyStrategy = DistCpConstants.UNIFORMSIZE; + /** The path to a file containing a list of paths to filter out of copy. */ + private final String filtersFile; - private EnumSet preserveStatus = EnumSet.noneOf(FileAttribute.class); + /** Path where output logs are stored. If not specified, it will use the + * default value JobStagingDir/_logs and delete upon job completion. */ + private final Path logPath; - private boolean preserveRawXattrs; + /** Set the copy strategy to use. Should map to a strategy implementation + * in distp-default.xml. */ + private final String copyStrategy; - private Path atomicWorkPath; + /** per map bandwidth in MB. */ + private final float mapBandwidth; - private Path logPath; + /** The number of threads to use for listStatus. We allow max + * {@link #MAX_NUM_LISTSTATUS_THREADS} threads. Setting numThreads to zero + * signify we should use the value from conf properties. */ + private final int numListstatusThreads; - private Path sourceFileListing; - private List sourcePaths; + /** The max number of maps to use for copy. */ + private final int maxMaps; - private String fromSnapshot; - private String toSnapshot; - - private Path targetPath; - - /** - * The path to a file containing a list of paths to filter out of the copy. - */ - private String filtersFile; - - // targetPathExist is a derived field, it's initialized in the - // beginning of distcp. - private boolean targetPathExists = true; + /** File attributes that need to be preserved. */ + private final EnumSet preserveStatus; // Size of chunk in number of blocks when splitting large file into chunks // to copy in parallel. Default is 0 and file are not splitted. - private int blocksPerChunk = 0; + private final int blocksPerChunk; - public static enum FileAttribute{ - REPLICATION, BLOCKSIZE, USER, GROUP, PERMISSION, CHECKSUMTYPE, ACL, XATTR, TIMES; + /** + * File attributes for preserve. + * + * Each enum entry uses the first char as its symbol. + */ + public enum FileAttribute { + REPLICATION, // R + BLOCKSIZE, // B + USER, // U + GROUP, // G + PERMISSION, // P + CHECKSUMTYPE, // C + ACL, // A + XATTR, // X + TIMES; // T public static FileAttribute getAttribute(char symbol) { for (FileAttribute attribute : values()) { @@ -115,187 +169,86 @@ public class DistCpOptions { } } - /** - * Constructor, to initialize source/target paths. - * @param sourcePaths List of source-paths (including wildcards) - * to be copied to target. - * @param targetPath Destination path for the dist-copy. - */ - public DistCpOptions(List sourcePaths, Path targetPath) { - assert sourcePaths != null && !sourcePaths.isEmpty() : "Invalid source paths"; - assert targetPath != null : "Invalid Target path"; + private DistCpOptions(Builder builder) { + this.sourceFileListing = builder.sourceFileListing; + this.sourcePaths = builder.sourcePaths; + this.targetPath = builder.targetPath; - this.sourcePaths = sourcePaths; - this.targetPath = targetPath; + this.atomicCommit = builder.atomicCommit; + this.atomicWorkPath = builder.atomicWorkPath; + this.syncFolder = builder.syncFolder; + this.deleteMissing = builder.deleteMissing; + this.ignoreFailures = builder.ignoreFailures; + this.overwrite = builder.overwrite; + this.append = builder.append; + this.skipCRC = builder.skipCRC; + this.blocking = builder.blocking; + + this.useDiff = builder.useDiff; + this.useRdiff = builder.useRdiff; + this.fromSnapshot = builder.fromSnapshot; + this.toSnapshot = builder.toSnapshot; + + this.filtersFile = builder.filtersFile; + this.logPath = builder.logPath; + this.copyStrategy = builder.copyStrategy; + + this.mapBandwidth = builder.mapBandwidth; + this.numListstatusThreads = builder.numListstatusThreads; + this.maxMaps = builder.maxMaps; + + this.preserveStatus = builder.preserveStatus; + + this.blocksPerChunk = builder.blocksPerChunk; } - /** - * Constructor, to initialize source/target paths. - * @param sourceFileListing File containing list of source paths - * @param targetPath Destination path for the dist-copy. - */ - public DistCpOptions(Path sourceFileListing, Path targetPath) { - assert sourceFileListing != null : "Invalid source paths"; - assert targetPath != null : "Invalid Target path"; - - this.sourceFileListing = sourceFileListing; - this.targetPath = targetPath; + public Path getSourceFileListing() { + return sourceFileListing; } - /** - * Copy constructor. - * @param that DistCpOptions being copied from. - */ - public DistCpOptions(DistCpOptions that) { - if (this != that && that != null) { - this.atomicCommit = that.atomicCommit; - this.syncFolder = that.syncFolder; - this.deleteMissing = that.deleteMissing; - this.ignoreFailures = that.ignoreFailures; - this.overwrite = that.overwrite; - this.skipCRC = that.skipCRC; - this.blocking = that.blocking; - this.useDiff = that.useDiff; - this.useRdiff = that.useRdiff; - this.numListstatusThreads = that.numListstatusThreads; - this.maxMaps = that.maxMaps; - this.mapBandwidth = that.mapBandwidth; - this.copyStrategy = that.copyStrategy; - this.preserveStatus = that.preserveStatus; - this.preserveRawXattrs = that.preserveRawXattrs; - this.atomicWorkPath = that.getAtomicWorkPath(); - this.logPath = that.getLogPath(); - this.sourceFileListing = that.getSourceFileListing(); - this.sourcePaths = that.getSourcePaths(); - this.targetPath = that.getTargetPath(); - this.targetPathExists = that.getTargetPathExists(); - this.filtersFile = that.getFiltersFile(); - this.blocksPerChunk = that.blocksPerChunk; - } + public List getSourcePaths() { + return sourcePaths == null ? + null : Collections.unmodifiableList(sourcePaths); + } + + public Path getTargetPath() { + return targetPath; } - /** - * Should the data be committed atomically? - * - * @return true if data should be committed automically. false otherwise - */ public boolean shouldAtomicCommit() { return atomicCommit; } - /** - * Set if data need to be committed automatically - * - * @param atomicCommit - boolean switch - */ - public void setAtomicCommit(boolean atomicCommit) { - this.atomicCommit = atomicCommit; + public Path getAtomicWorkPath() { + return atomicWorkPath; } - /** - * Should the data be sync'ed between source and target paths? - * - * @return true if data should be sync'ed up. false otherwise - */ public boolean shouldSyncFolder() { return syncFolder; } - /** - * Set if source and target folder contents be sync'ed up - * - * @param syncFolder - boolean switch - */ - public void setSyncFolder(boolean syncFolder) { - this.syncFolder = syncFolder; - } - - /** - * Should target files missing in source should be deleted? - * - * @return true if zoombie target files to be removed. false otherwise - */ public boolean shouldDeleteMissing() { return deleteMissing; } - /** - * Set if files only present in target should be deleted - * - * @param deleteMissing - boolean switch - */ - public void setDeleteMissing(boolean deleteMissing) { - this.deleteMissing = deleteMissing; - } - - /** - * Should failures be logged and ignored during copy? - * - * @return true if failures are to be logged and ignored. false otherwise - */ public boolean shouldIgnoreFailures() { return ignoreFailures; } - /** - * Set if failures during copy be ignored - * - * @param ignoreFailures - boolean switch - */ - public void setIgnoreFailures(boolean ignoreFailures) { - this.ignoreFailures = ignoreFailures; - } - - /** - * Should DistCp be running in blocking mode - * - * @return true if should run in blocking, false otherwise - */ - public boolean shouldBlock() { - return blocking; - } - - /** - * Set if Disctp should run blocking or non-blocking - * - * @param blocking - boolean switch - */ - public void setBlocking(boolean blocking) { - this.blocking = blocking; - } - - /** - * Should files be overwritten always? - * - * @return true if files in target that may exist before distcp, should always - * be overwritten. false otherwise - */ public boolean shouldOverwrite() { return overwrite; } - /** - * Set if files should always be overwritten on target - * - * @param overwrite - boolean switch - */ - public void setOverwrite(boolean overwrite) { - this.overwrite = overwrite; - } - - /** - * @return whether we can append new data to target files - */ public boolean shouldAppend() { return append; } - /** - * Set if we want to append new data to target files. This is valid only with - * update option and CRC is not skipped. - */ - public void setAppend(boolean append) { - this.append = append; + public boolean shouldSkipCRC() { + return skipCRC; + } + + public boolean shouldBlock() { + return blocking; } public boolean shouldUseDiff() { @@ -318,104 +271,34 @@ public class DistCpOptions { return this.toSnapshot; } - public void setUseDiff(String fromSS, String toSS) { - this.useDiff = true; - this.fromSnapshot = fromSS; - this.toSnapshot = toSS; + public String getFiltersFile() { + return filtersFile; } - public void setUseRdiff(String fromSS, String toSS) { - this.useRdiff = true; - this.fromSnapshot = fromSS; - this.toSnapshot = toSS; + public Path getLogPath() { + return logPath; } - /** - * Should CRC/checksum check be skipped while checking files are identical - * - * @return true if checksum check should be skipped while checking files are - * identical. false otherwise - */ - public boolean shouldSkipCRC() { - return skipCRC; + public String getCopyStrategy() { + return copyStrategy; } - /** - * Set if checksum comparison should be skipped while determining if - * source and destination files are identical - * - * @param skipCRC - boolean switch - */ - public void setSkipCRC(boolean skipCRC) { - this.skipCRC = skipCRC; - } - - /** Get the number of threads to use for listStatus - * - * @return Number of threads to do listStatus - */ public int getNumListstatusThreads() { return numListstatusThreads; } - /** Set the number of threads to use for listStatus. We allow max 40 - * threads. Setting numThreads to zero signify we should use the value - * from conf properties. - * - * @param numThreads - Number of threads - */ - public void setNumListstatusThreads(int numThreads) { - if (numThreads > maxNumListstatusThreads) { - this.numListstatusThreads = maxNumListstatusThreads; - } else if (numThreads > 0) { - this.numListstatusThreads = numThreads; - } else { - this.numListstatusThreads = 0; - } - } - - /** Get the max number of maps to use for this copy - * - * @return Max number of maps - */ public int getMaxMaps() { return maxMaps; } - /** - * Set the max number of maps to use for copy - * - * @param maxMaps - Number of maps - */ - public void setMaxMaps(int maxMaps) { - this.maxMaps = Math.max(maxMaps, 1); - } - - /** Get the map bandwidth in MB - * - * @return Bandwidth in MB - */ public float getMapBandwidth() { return mapBandwidth; } - /** - * Set per map bandwidth - * - * @param mapBandwidth - per map bandwidth - */ - public void setMapBandwidth(float mapBandwidth) { - assert mapBandwidth > 0 : "Bandwidth " + mapBandwidth + " is invalid (should be > 0)"; - this.mapBandwidth = mapBandwidth; - } - - /** - * Returns an iterator with the list of file attributes to preserve - * - * @return iterator of file attributes to preserve - */ - public Iterator preserveAttributes() { - return preserveStatus.iterator(); + public Set getPreserveAttributes() { + return (preserveStatus == null) + ? null + : Collections.unmodifiableSet(preserveStatus); } /** @@ -428,230 +311,10 @@ public class DistCpOptions { return preserveStatus.contains(attribute); } - /** - * Add file attributes that need to be preserved. This method may be - * called multiple times to add attributes. - * - * @param fileAttribute - Attribute to add, one at a time - */ - public void preserve(FileAttribute fileAttribute) { - for (FileAttribute attribute : preserveStatus) { - if (attribute.equals(fileAttribute)) { - return; - } - } - preserveStatus.add(fileAttribute); - } - - /** - * Return true if raw.* xattrs should be preserved. - * @return true if raw.* xattrs should be preserved. - */ - public boolean shouldPreserveRawXattrs() { - return preserveRawXattrs; - } - - /** - * Indicate that raw.* xattrs should be preserved - */ - public void preserveRawXattrs() { - preserveRawXattrs = true; - } - - /** Get work path for atomic commit. If null, the work - * path would be parentOf(targetPath) + "/._WIP_" + nameOf(targetPath) - * - * @return Atomic work path on the target cluster. Null if not set - */ - public Path getAtomicWorkPath() { - return atomicWorkPath; - } - - /** - * Set the work path for atomic commit - * - * @param atomicWorkPath - Path on the target cluster - */ - public void setAtomicWorkPath(Path atomicWorkPath) { - this.atomicWorkPath = atomicWorkPath; - } - - /** Get output directory for writing distcp logs. Otherwise logs - * are temporarily written to JobStagingDir/_logs and deleted - * upon job completion - * - * @return Log output path on the cluster where distcp job is run - */ - public Path getLogPath() { - return logPath; - } - - /** - * Set the log path where distcp output logs are stored - * Uses JobStagingDir/_logs by default - * - * @param logPath - Path where logs will be saved - */ - public void setLogPath(Path logPath) { - this.logPath = logPath; - } - - /** - * Get the copy strategy to use. Uses appropriate input format - * - * @return copy strategy to use - */ - public String getCopyStrategy() { - return copyStrategy; - } - - /** - * Set the copy strategy to use. Should map to a strategy implementation - * in distp-default.xml - * - * @param copyStrategy - copy Strategy to use - */ - public void setCopyStrategy(String copyStrategy) { - this.copyStrategy = copyStrategy; - } - - /** - * File path (hdfs:// or file://) that contains the list of actual - * files to copy - * - * @return - Source listing file path - */ - public Path getSourceFileListing() { - return sourceFileListing; - } - - /** - * Getter for sourcePaths. - * @return List of source-paths. - */ - public List getSourcePaths() { - return sourcePaths; - } - - /** - * Setter for sourcePaths. - * @param sourcePaths The new list of source-paths. - */ - public void setSourcePaths(List sourcePaths) { - assert sourcePaths != null && sourcePaths.size() != 0; - this.sourcePaths = sourcePaths; - } - - /** - * Getter for the targetPath. - * @return The target-path. - */ - public Path getTargetPath() { - return targetPath; - } - - /** - * Getter for the targetPathExists. - * @return The target-path. - */ - public boolean getTargetPathExists() { - return targetPathExists; - } - - /** - * Set targetPathExists. - * @param targetPathExists Whether the target path of distcp exists. - */ - public boolean setTargetPathExists(boolean targetPathExists) { - return this.targetPathExists = targetPathExists; - } - - /** - * File path that contains the list of patterns - * for paths to be filtered from the file copy. - * @return - Filter file path. - */ - public final String getFiltersFile() { - return filtersFile; - } - - /** - * Set filtersFile. - * @param filtersFilename The path to a list of patterns to exclude from copy. - */ - public final void setFiltersFile(String filtersFilename) { - this.filtersFile = filtersFilename; - } - - public final void setBlocksPerChunk(int csize) { - this.blocksPerChunk = csize; - } - - public final int getBlocksPerChunk() { + public int getBlocksPerChunk() { return blocksPerChunk; } - public final boolean splitLargeFile() { - return blocksPerChunk > 0; - } - - void validate() { - if ((useDiff || useRdiff) && deleteMissing) { - // -delete and -diff/-rdiff are mutually exclusive. For backward - // compatibility, we ignore the -delete option here, instead of throwing - // an IllegalArgumentException. See HDFS-10397 for more discussion. - OptionsParser.LOG.warn( - "-delete and -diff/-rdiff are mutually exclusive. " + - "The -delete option will be ignored."); - setDeleteMissing(false); - } - - if (syncFolder && atomicCommit) { - throw new IllegalArgumentException("Atomic commit can't be used with " + - "sync folder or overwrite options"); - } - - if (deleteMissing && !(overwrite || syncFolder)) { - throw new IllegalArgumentException("Delete missing is applicable " + - "only with update or overwrite options"); - } - - if (overwrite && syncFolder) { - throw new IllegalArgumentException("Overwrite and update options are " + - "mutually exclusive"); - } - - if (!syncFolder && skipCRC) { - throw new IllegalArgumentException("Skip CRC is valid only with update options"); - } - - if (!syncFolder && append) { - throw new IllegalArgumentException( - "Append is valid only with update options"); - } - if (skipCRC && append) { - throw new IllegalArgumentException( - "Append is disallowed when skipping CRC"); - } - if (!syncFolder && (useDiff || useRdiff)) { - throw new IllegalArgumentException( - "-diff/-rdiff is valid only with -update option"); - } - - if (useDiff || useRdiff) { - if (StringUtils.isBlank(fromSnapshot) || - StringUtils.isBlank(toSnapshot)) { - throw new IllegalArgumentException( - "Must provide both the starting and ending " + - "snapshot names for -diff/-rdiff"); - } - } - if (useDiff && useRdiff) { - throw new IllegalArgumentException( - "-diff and -rdiff are mutually exclusive"); - } - } - /** * Add options to configuration. These will be used in the Mapper/committer * @@ -715,20 +378,292 @@ public class DistCpOptions { ", mapBandwidth=" + mapBandwidth + ", copyStrategy='" + copyStrategy + '\'' + ", preserveStatus=" + preserveStatus + - ", preserveRawXattrs=" + preserveRawXattrs + ", atomicWorkPath=" + atomicWorkPath + ", logPath=" + logPath + ", sourceFileListing=" + sourceFileListing + ", sourcePaths=" + sourcePaths + ", targetPath=" + targetPath + - ", targetPathExists=" + targetPathExists + ", filtersFile='" + filtersFile + '\'' + ", blocksPerChunk=" + blocksPerChunk + '}'; } - @Override - protected DistCpOptions clone() throws CloneNotSupportedException { - return (DistCpOptions) super.clone(); + /** + * The builder of the {@link DistCpOptions}. + * + * This is designed to be the only public interface to create a + * {@link DistCpOptions} object for users. It follows a simple Builder design + * pattern. + */ + public static class Builder { + private Path sourceFileListing; + private List sourcePaths; + private Path targetPath; + + private boolean atomicCommit = false; + private Path atomicWorkPath; + private boolean syncFolder = false; + private boolean deleteMissing = false; + private boolean ignoreFailures = false; + private boolean overwrite = false; + private boolean append = false; + private boolean skipCRC = false; + private boolean blocking = true; + + private boolean useDiff = false; + private boolean useRdiff = false; + private String fromSnapshot; + private String toSnapshot; + + private String filtersFile; + + private Path logPath; + private String copyStrategy = DistCpConstants.UNIFORMSIZE; + + private int numListstatusThreads = 0; // 0 indicates that flag is not set. + private int maxMaps = DistCpConstants.DEFAULT_MAPS; + private float mapBandwidth = 0; // 0 indicates we should use the default + + private EnumSet preserveStatus = + EnumSet.noneOf(FileAttribute.class); + + private int blocksPerChunk = 0; + + public Builder(List sourcePaths, Path targetPath) { + Preconditions.checkArgument(sourcePaths != null && !sourcePaths.isEmpty(), + "Source paths should not be null or empty!"); + Preconditions.checkArgument(targetPath != null, + "Target path should not be null!"); + this.sourcePaths = sourcePaths; + this.targetPath = targetPath; + } + + public Builder(Path sourceFileListing, Path targetPath) { + Preconditions.checkArgument(sourceFileListing != null, + "Source file listing should not be null!"); + Preconditions.checkArgument(targetPath != null, + "Target path should not be null!"); + + this.sourceFileListing = sourceFileListing; + this.targetPath = targetPath; + } + + /** + * This is the single entry point for constructing DistCpOptions objects. + * + * Before a new DistCpOptions object is returned, it will set the dependent + * options, validate the option combinations. After constructing, the + * DistCpOptions instance is immutable. + */ + public DistCpOptions build() { + setOptionsForSplitLargeFile(); + + validate(); + + return new DistCpOptions(this); + } + + /** + * Override options for split large files. + */ + private void setOptionsForSplitLargeFile() { + if (blocksPerChunk <= 0) { + return; + } + + LOG.info("Enabling preserving blocksize since " + + DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch() + " is passed."); + preserve(FileAttribute.BLOCKSIZE); + + LOG.info("Set " + DistCpOptionSwitch.APPEND.getSwitch() + + " to false since " + DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch() + + " is passed."); + this.append = false; + } + + private void validate() { + if ((useDiff || useRdiff) && deleteMissing) { + // -delete and -diff/-rdiff are mutually exclusive. + throw new IllegalArgumentException("-delete and -diff/-rdiff are " + + "mutually exclusive. The -delete option will be ignored."); + } + + if (!atomicCommit && atomicWorkPath != null) { + throw new IllegalArgumentException( + "-tmp work-path can only be specified along with -atomic"); + } + + if (syncFolder && atomicCommit) { + throw new IllegalArgumentException("Atomic commit can't be used with " + + "sync folder or overwrite options"); + } + + if (deleteMissing && !(overwrite || syncFolder)) { + throw new IllegalArgumentException("Delete missing is applicable " + + "only with update or overwrite options"); + } + + if (overwrite && syncFolder) { + throw new IllegalArgumentException("Overwrite and update options are " + + "mutually exclusive"); + } + + if (!syncFolder && skipCRC) { + throw new IllegalArgumentException( + "Skip CRC is valid only with update options"); + } + + if (!syncFolder && append) { + throw new IllegalArgumentException( + "Append is valid only with update options"); + } + if (skipCRC && append) { + throw new IllegalArgumentException( + "Append is disallowed when skipping CRC"); + } + if (!syncFolder && (useDiff || useRdiff)) { + throw new IllegalArgumentException( + "-diff/-rdiff is valid only with -update option"); + } + + if (useDiff || useRdiff) { + if (StringUtils.isBlank(fromSnapshot) || + StringUtils.isBlank(toSnapshot)) { + throw new IllegalArgumentException( + "Must provide both the starting and ending " + + "snapshot names for -diff/-rdiff"); + } + } + if (useDiff && useRdiff) { + throw new IllegalArgumentException( + "-diff and -rdiff are mutually exclusive"); + } + } + + @VisibleForTesting + Builder withSourcePaths(List newSourcePaths) { + this.sourcePaths = newSourcePaths; + return this; + } + + public Builder withAtomicCommit(boolean newAtomicCommit) { + this.atomicCommit = newAtomicCommit; + return this; + } + + public Builder withAtomicWorkPath(Path newAtomicWorkPath) { + this.atomicWorkPath = newAtomicWorkPath; + return this; + } + + public Builder withSyncFolder(boolean newSyncFolder) { + this.syncFolder = newSyncFolder; + return this; + } + + public Builder withDeleteMissing(boolean newDeleteMissing) { + this.deleteMissing = newDeleteMissing; + return this; + } + + public Builder withIgnoreFailures(boolean newIgnoreFailures) { + this.ignoreFailures = newIgnoreFailures; + return this; + } + + public Builder withOverwrite(boolean newOverwrite) { + this.overwrite = newOverwrite; + return this; + } + + public Builder withAppend(boolean newAppend) { + this.append = newAppend; + return this; + } + + public Builder withCRC(boolean newSkipCRC) { + this.skipCRC = newSkipCRC; + return this; + } + + public Builder withBlocking(boolean newBlocking) { + this.blocking = newBlocking; + return this; + } + + public Builder withUseDiff(String newFromSnapshot, String newToSnapshot) { + this.useDiff = true; + this.fromSnapshot = newFromSnapshot; + this.toSnapshot = newToSnapshot; + return this; + } + + public Builder withUseRdiff(String newFromSnapshot, String newToSnapshot) { + this.useRdiff = true; + this.fromSnapshot = newFromSnapshot; + this.toSnapshot = newToSnapshot; + return this; + } + + public Builder withFiltersFile(String newFiletersFile) { + this.filtersFile = newFiletersFile; + return this; + } + + public Builder withLogPath(Path newLogPath) { + this.logPath = newLogPath; + return this; + } + + public Builder withCopyStrategy(String newCopyStrategy) { + this.copyStrategy = newCopyStrategy; + return this; + } + + public Builder withMapBandwidth(float newMapBandwidth) { + Preconditions.checkArgument(newMapBandwidth > 0, + "Bandwidth " + newMapBandwidth + " is invalid (should be > 0)"); + this.mapBandwidth = newMapBandwidth; + return this; + } + + public Builder withNumListstatusThreads(int newNumListstatusThreads) { + if (newNumListstatusThreads > MAX_NUM_LISTSTATUS_THREADS) { + this.numListstatusThreads = MAX_NUM_LISTSTATUS_THREADS; + } else if (newNumListstatusThreads > 0) { + this.numListstatusThreads = newNumListstatusThreads; + } else { + this.numListstatusThreads = 0; + } + return this; + } + + public Builder maxMaps(int newMaxMaps) { + this.maxMaps = Math.max(newMaxMaps, 1); + return this; + } + + public Builder preserve(String attributes) { + if (attributes == null || attributes.isEmpty()) { + preserveStatus = EnumSet.allOf(FileAttribute.class); + } else { + for (int index = 0; index < attributes.length(); index++) { + preserveStatus.add(FileAttribute. + getAttribute(attributes.charAt(index))); + } + } + return this; + } + + public Builder preserve(FileAttribute attribute) { + preserveStatus.add(attribute); + return this; + } + + public Builder withBlocksPerChunk(int newBlocksPerChunk) { + this.blocksPerChunk = newBlocksPerChunk; + return this; + } } + } diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java index bcae96a8d83..a78320b05bf 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java @@ -48,7 +48,7 @@ import java.util.HashSet; * source.s1 */ class DistCpSync { - private DistCpOptions inputOptions; + private DistCpContext context; private Configuration conf; // diffMap maps snapshot diff op type to a list of diff ops. // It's initially created based on the snapshot diff. Then the individual @@ -58,13 +58,13 @@ class DistCpSync { private EnumMap> diffMap; private DiffInfo[] renameDiffs; - DistCpSync(DistCpOptions options, Configuration conf) { - this.inputOptions = options; + DistCpSync(DistCpContext context, Configuration conf) { + this.context = context; this.conf = conf; } private boolean isRdiff() { - return inputOptions.shouldUseRdiff(); + return context.shouldUseRdiff(); } /** @@ -77,14 +77,14 @@ class DistCpSync { * default distcp if the third condition isn't met. */ private boolean preSyncCheck() throws IOException { - List sourcePaths = inputOptions.getSourcePaths(); + List sourcePaths = context.getSourcePaths(); if (sourcePaths.size() != 1) { // we only support one source dir which must be a snapshottable directory throw new IllegalArgumentException(sourcePaths.size() + " source paths are provided"); } final Path sourceDir = sourcePaths.get(0); - final Path targetDir = inputOptions.getTargetPath(); + final Path targetDir = context.getTargetPath(); final FileSystem srcFs = sourceDir.getFileSystem(conf); final FileSystem tgtFs = targetDir.getFileSystem(conf); @@ -104,13 +104,15 @@ class DistCpSync { // make sure targetFS has no change between from and the current states if (!checkNoChange(targetFs, targetDir)) { // set the source path using the snapshot path - inputOptions.setSourcePaths(Arrays.asList(getSnapshotPath(sourceDir, - inputOptions.getToSnapshot()))); + context.setSourcePaths(Arrays.asList(getSnapshotPath(sourceDir, + context.getToSnapshot()))); return false; } - final String from = getSnapshotName(inputOptions.getFromSnapshot()); - final String to = getSnapshotName(inputOptions.getToSnapshot()); + final String from = getSnapshotName( + context.getFromSnapshot()); + final String to = getSnapshotName( + context.getToSnapshot()); try { final FileStatus fromSnapshotStat = @@ -152,9 +154,9 @@ class DistCpSync { return false; } - List sourcePaths = inputOptions.getSourcePaths(); + List sourcePaths = context.getSourcePaths(); final Path sourceDir = sourcePaths.get(0); - final Path targetDir = inputOptions.getTargetPath(); + final Path targetDir = context.getTargetPath(); final FileSystem tfs = targetDir.getFileSystem(conf); final DistributedFileSystem targetFs = (DistributedFileSystem) tfs; @@ -175,8 +177,8 @@ class DistCpSync { deleteTargetTmpDir(targetFs, tmpDir); // TODO: since we have tmp directory, we can support "undo" with failures // set the source path using the snapshot path - inputOptions.setSourcePaths(Arrays.asList(getSnapshotPath(sourceDir, - inputOptions.getToSnapshot()))); + context.setSourcePaths(Arrays.asList(getSnapshotPath(sourceDir, + context.getToSnapshot()))); } } @@ -187,13 +189,13 @@ class DistCpSync { */ private boolean getAllDiffs() throws IOException { Path ssDir = isRdiff()? - inputOptions.getTargetPath() : inputOptions.getSourcePaths().get(0); + context.getTargetPath() : context.getSourcePaths().get(0); try { DistributedFileSystem fs = (DistributedFileSystem) ssDir.getFileSystem(conf); - final String from = getSnapshotName(inputOptions.getFromSnapshot()); - final String to = getSnapshotName(inputOptions.getToSnapshot()); + final String from = getSnapshotName(context.getFromSnapshot()); + final String to = getSnapshotName(context.getToSnapshot()); SnapshotDiffReport report = fs.getSnapshotDiffReport(ssDir, from, to); this.diffMap = new EnumMap<>(SnapshotDiffReport.DiffType.class); @@ -273,19 +275,19 @@ class DistCpSync { */ private boolean checkNoChange(DistributedFileSystem fs, Path path) { try { - final String from = getSnapshotName(inputOptions.getFromSnapshot()); + final String from = getSnapshotName(context.getFromSnapshot()); SnapshotDiffReport targetDiff = fs.getSnapshotDiffReport(path, from, ""); if (!targetDiff.getDiffList().isEmpty()) { DistCp.LOG.warn("The target has been modified since snapshot " - + inputOptions.getFromSnapshot()); + + context.getFromSnapshot()); return false; } else { return true; } } catch (IOException e) { DistCp.LOG.warn("Failed to compute snapshot diff on " + path - + " at snapshot " + inputOptions.getFromSnapshot(), e); + + " at snapshot " + context.getFromSnapshot(), e); } return false; } diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/FileBasedCopyListing.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/FileBasedCopyListing.java index 2bc343e1727..c356edd4251 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/FileBasedCopyListing.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/FileBasedCopyListing.java @@ -52,7 +52,7 @@ public class FileBasedCopyListing extends CopyListing { /** {@inheritDoc} */ @Override - protected void validatePaths(DistCpOptions options) + protected void validatePaths(DistCpContext context) throws IOException, InvalidInputException { } @@ -60,14 +60,14 @@ public class FileBasedCopyListing extends CopyListing { * Implementation of CopyListing::buildListing(). * Iterates over all source paths mentioned in the input-file. * @param pathToListFile Path on HDFS where the listing file is written. - * @param options Input Options for DistCp (indicating source/target paths.) + * @param context Distcp context with associated input options. * @throws IOException */ @Override - public void doBuildListing(Path pathToListFile, DistCpOptions options) throws IOException { - DistCpOptions newOption = new DistCpOptions(options); - newOption.setSourcePaths(fetchFileList(options.getSourceFileListing())); - globbedListing.buildListing(pathToListFile, newOption); + public void doBuildListing(Path pathToListFile, DistCpContext context) + throws IOException { + context.setSourcePaths(fetchFileList(context.getSourceFileListing())); + globbedListing.buildListing(pathToListFile, context); } private List fetchFileList(Path sourceListing) throws IOException { diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/GlobbedCopyListing.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/GlobbedCopyListing.java index 27330b78f72..63c6f436e59 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/GlobbedCopyListing.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/GlobbedCopyListing.java @@ -51,7 +51,7 @@ public class GlobbedCopyListing extends CopyListing { /** {@inheritDoc} */ @Override - protected void validatePaths(DistCpOptions options) + protected void validatePaths(DistCpContext context) throws IOException, InvalidInputException { } @@ -60,19 +60,19 @@ public class GlobbedCopyListing extends CopyListing { * Creates the copy listing by "globbing" all source-paths. * @param pathToListingFile The location at which the copy-listing file * is to be created. - * @param options Input Options for DistCp (indicating source/target paths.) + * @param context The distcp context with associated input options. * @throws IOException */ @Override - public void doBuildListing(Path pathToListingFile, - DistCpOptions options) throws IOException { + public void doBuildListing(Path pathToListingFile, DistCpContext context) + throws IOException { List globbedPaths = new ArrayList(); - if (options.getSourcePaths().isEmpty()) { + if (context.getSourcePaths().isEmpty()) { throw new InvalidInputException("Nothing to process. Source paths::EMPTY"); } - for (Path p : options.getSourcePaths()) { + for (Path p : context.getSourcePaths()) { FileSystem fs = p.getFileSystem(getConf()); FileStatus[] inputs = fs.globStatus(p); @@ -85,9 +85,8 @@ public class GlobbedCopyListing extends CopyListing { } } - DistCpOptions optionsGlobbed = new DistCpOptions(options); - optionsGlobbed.setSourcePaths(globbedPaths); - simpleListing.buildListing(pathToListingFile, optionsGlobbed); + context.setSourcePaths(globbedPaths); + simpleListing.buildListing(pathToListingFile, context); } /** {@inheritDoc} */ diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java index 8881264d281..21ff0f86841 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java @@ -32,7 +32,6 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.tools.DistCpOptions.FileAttribute; import com.google.common.base.Preconditions; @@ -95,253 +94,126 @@ public class OptionsParser { Arrays.toString(args), e); } - DistCpOptions option = parseSourceAndTargetPaths(command); - - option.setIgnoreFailures( - command.hasOption(DistCpOptionSwitch.IGNORE_FAILURES.getSwitch())); - - option.setAtomicCommit( - command.hasOption(DistCpOptionSwitch.ATOMIC_COMMIT.getSwitch())); - - option.setSyncFolder( - command.hasOption(DistCpOptionSwitch.SYNC_FOLDERS.getSwitch())); - - option.setOverwrite( - command.hasOption(DistCpOptionSwitch.OVERWRITE.getSwitch())); - - option.setAppend( - command.hasOption(DistCpOptionSwitch.APPEND.getSwitch())); - - option.setDeleteMissing( - command.hasOption(DistCpOptionSwitch.DELETE_MISSING.getSwitch())); - - option.setSkipCRC( - command.hasOption(DistCpOptionSwitch.SKIP_CRC.getSwitch())); - - if (command.hasOption(DistCpOptionSwitch.WORK_PATH.getSwitch()) && - option.shouldAtomicCommit()) { - String workPath = getVal(command, DistCpOptionSwitch.WORK_PATH.getSwitch()); - if (workPath != null && !workPath.isEmpty()) { - option.setAtomicWorkPath(new Path(workPath)); - } - } else if (command.hasOption(DistCpOptionSwitch.WORK_PATH.getSwitch())) { - throw new IllegalArgumentException("-tmp work-path can only be specified along with -atomic"); - } - - if (command.hasOption(DistCpOptionSwitch.LOG_PATH.getSwitch())) { - option.setLogPath(new Path(getVal(command, DistCpOptionSwitch.LOG_PATH.getSwitch()))); - } - - - if (command.hasOption(DistCpOptionSwitch.BLOCKING.getSwitch())) { - option.setBlocking(false); - } - - parseBandwidth(command, option); - - parseNumListStatusThreads(command, option); - - parseMaxMaps(command, option); - - if (command.hasOption(DistCpOptionSwitch.COPY_STRATEGY.getSwitch())) { - option.setCopyStrategy( - getVal(command, DistCpOptionSwitch.COPY_STRATEGY.getSwitch())); - } - - parsePreserveStatus(command, option); + DistCpOptions.Builder builder = parseSourceAndTargetPaths(command); + builder + .withAtomicCommit( + command.hasOption(DistCpOptionSwitch.ATOMIC_COMMIT.getSwitch())) + .withSyncFolder( + command.hasOption(DistCpOptionSwitch.SYNC_FOLDERS.getSwitch())) + .withDeleteMissing( + command.hasOption(DistCpOptionSwitch.DELETE_MISSING.getSwitch())) + .withIgnoreFailures( + command.hasOption(DistCpOptionSwitch.IGNORE_FAILURES.getSwitch())) + .withOverwrite( + command.hasOption(DistCpOptionSwitch.OVERWRITE.getSwitch())) + .withAppend( + command.hasOption(DistCpOptionSwitch.APPEND.getSwitch())) + .withCRC( + command.hasOption(DistCpOptionSwitch.SKIP_CRC.getSwitch())) + .withBlocking( + !command.hasOption(DistCpOptionSwitch.BLOCKING.getSwitch())); if (command.hasOption(DistCpOptionSwitch.DIFF.getSwitch())) { String[] snapshots = getVals(command, DistCpOptionSwitch.DIFF.getSwitch()); checkSnapshotsArgs(snapshots); - option.setUseDiff(snapshots[0], snapshots[1]); + builder.withUseDiff(snapshots[0], snapshots[1]); } if (command.hasOption(DistCpOptionSwitch.RDIFF.getSwitch())) { String[] snapshots = getVals(command, DistCpOptionSwitch.RDIFF.getSwitch()); checkSnapshotsArgs(snapshots); - option.setUseRdiff(snapshots[0], snapshots[1]); + builder.withUseRdiff(snapshots[0], snapshots[1]); } - parseFileLimit(command); - - parseSizeLimit(command); - if (command.hasOption(DistCpOptionSwitch.FILTERS.getSwitch())) { - option.setFiltersFile(getVal(command, - DistCpOptionSwitch.FILTERS.getSwitch())); + builder.withFiltersFile( + getVal(command, DistCpOptionSwitch.FILTERS.getSwitch())); } - parseBlocksPerChunk(command, option); + if (command.hasOption(DistCpOptionSwitch.LOG_PATH.getSwitch())) { + builder.withLogPath( + new Path(getVal(command, DistCpOptionSwitch.LOG_PATH.getSwitch()))); + } - option.validate(); - - return option; - } - - - /** - * A helper method to parse chunk size in number of blocks. - * Used when breaking large file into chunks to copy in parallel. - * - * @param command command line arguments - */ - private static void parseBlocksPerChunk(CommandLine command, - DistCpOptions option) { - boolean hasOption = - command.hasOption(DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch()); - LOG.info("parseChunkSize: " + - DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch() + " " + hasOption); - if (hasOption) { - String chunkSizeString = getVal(command, - DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch().trim()); - try { - int csize = Integer.parseInt(chunkSizeString); - if (csize < 0) { - csize = 0; - } - LOG.info("Set distcp blocksPerChunk to " + csize); - option.setBlocksPerChunk(csize); - } - catch (NumberFormatException e) { - throw new IllegalArgumentException("blocksPerChunk is invalid: " - + chunkSizeString, e); + if (command.hasOption(DistCpOptionSwitch.WORK_PATH.getSwitch())) { + final String workPath = getVal(command, + DistCpOptionSwitch.WORK_PATH.getSwitch()); + if (workPath != null && !workPath.isEmpty()) { + builder.withAtomicWorkPath(new Path(workPath)); } } - } - /** - * parseSizeLimit is a helper method for parsing the deprecated - * argument SIZE_LIMIT. - * - * @param command command line arguments - */ - private static void parseSizeLimit(CommandLine command) { - if (command.hasOption(DistCpOptionSwitch.SIZE_LIMIT.getSwitch())) { - String sizeLimitString = getVal(command, - DistCpOptionSwitch.SIZE_LIMIT.getSwitch().trim()); + if (command.hasOption(DistCpOptionSwitch.BANDWIDTH.getSwitch())) { try { - Long.parseLong(sizeLimitString); - } - catch (NumberFormatException e) { - throw new IllegalArgumentException("Size-limit is invalid: " - + sizeLimitString, e); - } - LOG.warn(DistCpOptionSwitch.SIZE_LIMIT.getSwitch() + " is a deprecated" + - " option. Ignoring."); - } - } - - /** - * parseFileLimit is a helper method for parsing the deprecated - * argument FILE_LIMIT. - * - * @param command command line arguments - */ - private static void parseFileLimit(CommandLine command) { - if (command.hasOption(DistCpOptionSwitch.FILE_LIMIT.getSwitch())) { - String fileLimitString = getVal(command, - DistCpOptionSwitch.FILE_LIMIT.getSwitch().trim()); - try { - Integer.parseInt(fileLimitString); + final Float mapBandwidth = Float.parseFloat( + getVal(command, DistCpOptionSwitch.BANDWIDTH.getSwitch())); + builder.withMapBandwidth(mapBandwidth); } catch (NumberFormatException e) { - throw new IllegalArgumentException("File-limit is invalid: " - + fileLimitString, e); - } - LOG.warn(DistCpOptionSwitch.FILE_LIMIT.getSwitch() + " is a deprecated" + - " option. Ignoring."); - } - } - - /** - * parsePreserveStatus is a helper method for parsing PRESERVE_STATUS. - * - * @param command command line arguments - * @param option parsed distcp options - */ - private static void parsePreserveStatus(CommandLine command, - DistCpOptions option) { - if (command.hasOption(DistCpOptionSwitch.PRESERVE_STATUS.getSwitch())) { - String attributes = - getVal(command, DistCpOptionSwitch.PRESERVE_STATUS.getSwitch()); - if (attributes == null || attributes.isEmpty()) { - for (FileAttribute attribute : FileAttribute.values()) { - option.preserve(attribute); - } - } else { - for (int index = 0; index < attributes.length(); index++) { - option.preserve(FileAttribute. - getAttribute(attributes.charAt(index))); - } + throw new IllegalArgumentException("Bandwidth specified is invalid: " + + getVal(command, DistCpOptionSwitch.BANDWIDTH.getSwitch()), e); } } - } - /** - * parseMaxMaps is a helper method for parsing MAX_MAPS. - * - * @param command command line arguments - * @param option parsed distcp options - */ - private static void parseMaxMaps(CommandLine command, - DistCpOptions option) { - if (command.hasOption(DistCpOptionSwitch.MAX_MAPS.getSwitch())) { - try { - Integer maps = Integer.parseInt( - getVal(command, DistCpOptionSwitch.MAX_MAPS.getSwitch()).trim()); - option.setMaxMaps(maps); - } catch (NumberFormatException e) { - throw new IllegalArgumentException("Number of maps is invalid: " + - getVal(command, DistCpOptionSwitch.MAX_MAPS.getSwitch()), e); - } - } - } - - /** - * parseNumListStatusThreads is a helper method for parsing - * NUM_LISTSTATUS_THREADS. - * - * @param command command line arguments - * @param option parsed distcp options - */ - private static void parseNumListStatusThreads(CommandLine command, - DistCpOptions option) { if (command.hasOption( DistCpOptionSwitch.NUM_LISTSTATUS_THREADS.getSwitch())) { try { - Integer numThreads = Integer.parseInt(getVal(command, - DistCpOptionSwitch.NUM_LISTSTATUS_THREADS.getSwitch()).trim()); - option.setNumListstatusThreads(numThreads); + final Integer numThreads = Integer.parseInt(getVal(command, + DistCpOptionSwitch.NUM_LISTSTATUS_THREADS.getSwitch())); + builder.withNumListstatusThreads(numThreads); } catch (NumberFormatException e) { throw new IllegalArgumentException( "Number of liststatus threads is invalid: " + getVal(command, DistCpOptionSwitch.NUM_LISTSTATUS_THREADS.getSwitch()), e); } } - } - /** - * parseBandwidth is a helper method for parsing BANDWIDTH. - * - * @param command command line arguments - * @param option parsed distcp options - */ - private static void parseBandwidth(CommandLine command, - DistCpOptions option) { - if (command.hasOption(DistCpOptionSwitch.BANDWIDTH.getSwitch())) { + if (command.hasOption(DistCpOptionSwitch.MAX_MAPS.getSwitch())) { try { - Float mapBandwidth = Float.parseFloat( - getVal(command, DistCpOptionSwitch.BANDWIDTH.getSwitch()).trim()); - if (mapBandwidth <= 0) { - throw new IllegalArgumentException("Bandwidth specified is not " + - "positive: " + mapBandwidth); - } - option.setMapBandwidth(mapBandwidth); + final Integer maps = Integer.parseInt( + getVal(command, DistCpOptionSwitch.MAX_MAPS.getSwitch())); + builder.maxMaps(maps); } catch (NumberFormatException e) { - throw new IllegalArgumentException("Bandwidth specified is invalid: " + - getVal(command, DistCpOptionSwitch.BANDWIDTH.getSwitch()), e); + throw new IllegalArgumentException("Number of maps is invalid: " + + getVal(command, DistCpOptionSwitch.MAX_MAPS.getSwitch()), e); } } + + if (command.hasOption(DistCpOptionSwitch.COPY_STRATEGY.getSwitch())) { + builder.withCopyStrategy( + getVal(command, DistCpOptionSwitch.COPY_STRATEGY.getSwitch())); + } + + if (command.hasOption(DistCpOptionSwitch.PRESERVE_STATUS.getSwitch())) { + builder.preserve( + getVal(command, DistCpOptionSwitch.PRESERVE_STATUS.getSwitch())); + } + + if (command.hasOption(DistCpOptionSwitch.FILE_LIMIT.getSwitch())) { + LOG.warn(DistCpOptionSwitch.FILE_LIMIT.getSwitch() + " is a deprecated" + + " option. Ignoring."); + } + + if (command.hasOption(DistCpOptionSwitch.SIZE_LIMIT.getSwitch())) { + LOG.warn(DistCpOptionSwitch.SIZE_LIMIT.getSwitch() + " is a deprecated" + + " option. Ignoring."); + } + + if (command.hasOption(DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch())) { + final String chunkSizeStr = getVal(command, + DistCpOptionSwitch.BLOCKS_PER_CHUNK.getSwitch().trim()); + try { + int csize = Integer.parseInt(chunkSizeStr); + csize = csize > 0 ? csize : 0; + LOG.info("Set distcp blocksPerChunk to " + csize); + builder.withBlocksPerChunk(csize); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("blocksPerChunk is invalid: " + + chunkSizeStr, e); + } + } + + return builder.build(); } /** @@ -351,9 +223,8 @@ public class OptionsParser { * @param command command line arguments * @return DistCpOptions */ - private static DistCpOptions parseSourceAndTargetPaths( + private static DistCpOptions.Builder parseSourceAndTargetPaths( CommandLine command) { - DistCpOptions option; Path targetPath; List sourcePaths = new ArrayList(); @@ -378,20 +249,22 @@ public class OptionsParser { throw new IllegalArgumentException("Both source file listing and " + "source paths present"); } - option = new DistCpOptions(new Path(getVal(command, DistCpOptionSwitch. - SOURCE_FILE_LISTING.getSwitch())), targetPath); + return new DistCpOptions.Builder(new Path(getVal(command, + DistCpOptionSwitch.SOURCE_FILE_LISTING.getSwitch())), targetPath); } else { if (sourcePaths.isEmpty()) { throw new IllegalArgumentException("Neither source file listing nor " + "source paths present"); } - option = new DistCpOptions(sourcePaths, targetPath); + return new DistCpOptions.Builder(sourcePaths, targetPath); } - return option; } private static String getVal(CommandLine command, String swtch) { - String optionValue = command.getOptionValue(swtch); + if (swtch == null) { + return null; + } + String optionValue = command.getOptionValue(swtch.trim()); if (optionValue == null) { return null; } else { diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java index af913474550..8111b047571 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java @@ -123,10 +123,10 @@ public class SimpleCopyListing extends CopyListing { } @Override - protected void validatePaths(DistCpOptions options) + protected void validatePaths(DistCpContext context) throws IOException, InvalidInputException { - Path targetPath = options.getTargetPath(); + Path targetPath = context.getTargetPath(); FileSystem targetFS = targetPath.getFileSystem(getConf()); boolean targetExists = false; boolean targetIsFile = false; @@ -142,12 +142,12 @@ public class SimpleCopyListing extends CopyListing { //If target is a file, then source has to be single file if (targetIsFile) { - if (options.getSourcePaths().size() > 1) { + if (context.getSourcePaths().size() > 1) { throw new InvalidInputException("Multiple source being copied to a file: " + targetPath); } - Path srcPath = options.getSourcePaths().get(0); + Path srcPath = context.getSourcePaths().get(0); FileSystem sourceFS = srcPath.getFileSystem(getConf()); if (!sourceFS.isFile(srcPath)) { throw new InvalidInputException("Cannot copy " + srcPath + @@ -155,12 +155,12 @@ public class SimpleCopyListing extends CopyListing { } } - if (options.shouldAtomicCommit() && targetExists) { + if (context.shouldAtomicCommit() && targetExists) { throw new InvalidInputException("Target path for atomic-commit already exists: " + targetPath + ". Cannot atomic-commit to pre-existing target-path."); } - for (Path path: options.getSourcePaths()) { + for (Path path: context.getSourcePaths()) { FileSystem fs = path.getFileSystem(getConf()); if (!fs.exists(path)) { throw new InvalidInputException(path + " doesn't exist"); @@ -184,7 +184,7 @@ public class SimpleCopyListing extends CopyListing { } if (targetIsReservedRaw) { - options.preserveRawXattrs(); + context.setPreserveRawXattrs(true); getConf().setBoolean(DistCpConstants.CONF_LABEL_PRESERVE_RAWXATTRS, true); } @@ -194,18 +194,19 @@ public class SimpleCopyListing extends CopyListing { */ Credentials credentials = getCredentials(); if (credentials != null) { - Path[] inputPaths = options.getSourcePaths().toArray(new Path[1]); + Path[] inputPaths = context.getSourcePaths() + .toArray(new Path[1]); TokenCache.obtainTokensForNamenodes(credentials, inputPaths, getConf()); } } @Override protected void doBuildListing(Path pathToListingFile, - DistCpOptions options) throws IOException { - if(options.shouldUseSnapshotDiff()) { - doBuildListingWithSnapshotDiff(getWriter(pathToListingFile), options); - }else { - doBuildListing(getWriter(pathToListingFile), options); + DistCpContext context) throws IOException { + if (context.shouldUseSnapshotDiff()) { + doBuildListingWithSnapshotDiff(getWriter(pathToListingFile), context); + } else { + doBuildListing(getWriter(pathToListingFile), context); } } @@ -232,22 +233,22 @@ public class SimpleCopyListing extends CopyListing { * @throws IOException */ private void addToFileListing(SequenceFile.Writer fileListWriter, - Path sourceRoot, Path path, DistCpOptions options) throws IOException { + Path sourceRoot, Path path, DistCpContext context) throws IOException { sourceRoot = getPathWithSchemeAndAuthority(sourceRoot); path = getPathWithSchemeAndAuthority(path); path = makeQualified(path); FileSystem sourceFS = sourceRoot.getFileSystem(getConf()); FileStatus fileStatus = sourceFS.getFileStatus(path); - final boolean preserveAcls = options.shouldPreserve(FileAttribute.ACL); - final boolean preserveXAttrs = options.shouldPreserve(FileAttribute.XATTR); - final boolean preserveRawXAttrs = options.shouldPreserveRawXattrs(); + final boolean preserveAcls = context.shouldPreserve(FileAttribute.ACL); + final boolean preserveXAttrs = context.shouldPreserve(FileAttribute.XATTR); + final boolean preserveRawXAttrs = context.shouldPreserveRawXattrs(); LinkedList fileCopyListingStatus = DistCpUtils.toCopyListingFileStatus(sourceFS, fileStatus, preserveAcls, preserveXAttrs, preserveRawXAttrs, - options.getBlocksPerChunk()); + context.getBlocksPerChunk()); writeToFileListingRoot(fileListWriter, fileCopyListingStatus, - sourceRoot, options); + sourceRoot, context); } /** @@ -258,14 +259,16 @@ public class SimpleCopyListing extends CopyListing { * {@link org.apache.hadoop.tools.DistCpSync#sync}. An item can be * created/modified and renamed, in which case, the target path is put * into the list. + * @param fileListWriter the list for holding processed results + * @param context The DistCp context with associated input options * @throws IOException */ @VisibleForTesting protected void doBuildListingWithSnapshotDiff( - SequenceFile.Writer fileListWriter, DistCpOptions options) + SequenceFile.Writer fileListWriter, DistCpContext context) throws IOException { ArrayList diffList = distCpSync.prepareDiffListForCopyListing(); - Path sourceRoot = options.getSourcePaths().get(0); + Path sourceRoot = context.getSourcePaths().get(0); FileSystem sourceFS = sourceRoot.getFileSystem(getConf()); try { @@ -273,13 +276,13 @@ public class SimpleCopyListing extends CopyListing { for (DiffInfo diff : diffList) { // add snapshot paths prefix diff.setTarget( - new Path(options.getSourcePaths().get(0), diff.getTarget())); + new Path(context.getSourcePaths().get(0), diff.getTarget())); if (diff.getType() == SnapshotDiffReport.DiffType.MODIFY) { addToFileListing(fileListWriter, - sourceRoot, diff.getTarget(), options); + sourceRoot, diff.getTarget(), context); } else if (diff.getType() == SnapshotDiffReport.DiffType.CREATE) { addToFileListing(fileListWriter, - sourceRoot, diff.getTarget(), options); + sourceRoot, diff.getTarget(), context); FileStatus sourceStatus = sourceFS.getFileStatus(diff.getTarget()); if (sourceStatus.isDirectory()) { @@ -290,13 +293,13 @@ public class SimpleCopyListing extends CopyListing { HashSet excludeList = distCpSync.getTraverseExcludeList(diff.getSource(), - options.getSourcePaths().get(0)); + context.getSourcePaths().get(0)); ArrayList sourceDirs = new ArrayList<>(); sourceDirs.add(sourceStatus); traverseDirectory(fileListWriter, sourceFS, sourceDirs, - sourceRoot, options, excludeList, fileStatuses); + sourceRoot, context, excludeList, fileStatuses); } } } @@ -325,27 +328,30 @@ public class SimpleCopyListing extends CopyListing { * See computeSourceRootPath method for how the root path of the source is * computed. * @param fileListWriter - * @param options + * @param context The distcp context with associated input options * @throws IOException */ @VisibleForTesting protected void doBuildListing(SequenceFile.Writer fileListWriter, - DistCpOptions options) throws IOException { - if (options.getNumListstatusThreads() > 0) { - numListstatusThreads = options.getNumListstatusThreads(); + DistCpContext context) throws IOException { + if (context.getNumListstatusThreads() > 0) { + numListstatusThreads = context.getNumListstatusThreads(); } try { List statusList = Lists.newArrayList(); - for (Path path: options.getSourcePaths()) { + for (Path path: context.getSourcePaths()) { FileSystem sourceFS = path.getFileSystem(getConf()); - final boolean preserveAcls = options.shouldPreserve(FileAttribute.ACL); - final boolean preserveXAttrs = options.shouldPreserve(FileAttribute.XATTR); - final boolean preserveRawXAttrs = options.shouldPreserveRawXattrs(); + final boolean preserveAcls = + context.shouldPreserve(FileAttribute.ACL); + final boolean preserveXAttrs = + context.shouldPreserve(FileAttribute.XATTR); + final boolean preserveRawXAttrs = + context.shouldPreserveRawXattrs(); path = makeQualified(path); FileStatus rootStatus = sourceFS.getFileStatus(path); - Path sourcePathRoot = computeSourceRootPath(rootStatus, options); + Path sourcePathRoot = computeSourceRootPath(rootStatus, context); FileStatus[] sourceFiles = sourceFS.listStatus(path); boolean explore = (sourceFiles != null && sourceFiles.length > 0); @@ -353,9 +359,9 @@ public class SimpleCopyListing extends CopyListing { LinkedList rootCopyListingStatus = DistCpUtils.toCopyListingFileStatus(sourceFS, rootStatus, preserveAcls, preserveXAttrs, preserveRawXAttrs, - options.getBlocksPerChunk()); + context.getBlocksPerChunk()); writeToFileListingRoot(fileListWriter, rootCopyListingStatus, - sourcePathRoot, options); + sourcePathRoot, context); } if (explore) { ArrayList sourceDirs = new ArrayList(); @@ -368,7 +374,7 @@ public class SimpleCopyListing extends CopyListing { preserveAcls && sourceStatus.isDirectory(), preserveXAttrs && sourceStatus.isDirectory(), preserveRawXAttrs && sourceStatus.isDirectory(), - options.getBlocksPerChunk()); + context.getBlocksPerChunk()); for (CopyListingFileStatus fs : sourceCopyListingStatus) { if (randomizeFileListing) { addToFileListing(statusList, @@ -385,7 +391,7 @@ public class SimpleCopyListing extends CopyListing { } } traverseDirectory(fileListWriter, sourceFS, sourceDirs, - sourcePathRoot, options, null, statusList); + sourcePathRoot, context, null, statusList); } } if (randomizeFileListing) { @@ -447,13 +453,13 @@ public class SimpleCopyListing extends CopyListing { } private Path computeSourceRootPath(FileStatus sourceStatus, - DistCpOptions options) throws IOException { + DistCpContext context) throws IOException { - Path target = options.getTargetPath(); + Path target = context.getTargetPath(); FileSystem targetFS = target.getFileSystem(getConf()); - final boolean targetPathExists = options.getTargetPathExists(); + final boolean targetPathExists = context.isTargetPathExists(); - boolean solitaryFile = options.getSourcePaths().size() == 1 + boolean solitaryFile = context.getSourcePaths().size() == 1 && !sourceStatus.isDirectory(); if (solitaryFile) { @@ -463,8 +469,11 @@ public class SimpleCopyListing extends CopyListing { return sourceStatus.getPath().getParent(); } } else { - boolean specialHandling = (options.getSourcePaths().size() == 1 && !targetPathExists) || - options.shouldSyncFolder() || options.shouldOverwrite(); + boolean specialHandling = + (context.getSourcePaths().size() == 1 && + !targetPathExists) || + context.shouldSyncFolder() || + context.shouldOverwrite(); if ((specialHandling && sourceStatus.isDirectory()) || sourceStatus.getPath().isRoot()) { @@ -610,13 +619,13 @@ public class SimpleCopyListing extends CopyListing { FileSystem sourceFS, ArrayList sourceDirs, Path sourcePathRoot, - DistCpOptions options, + DistCpContext context, HashSet excludeList, List fileStatuses) throws IOException { - final boolean preserveAcls = options.shouldPreserve(FileAttribute.ACL); - final boolean preserveXAttrs = options.shouldPreserve(FileAttribute.XATTR); - final boolean preserveRawXattrs = options.shouldPreserveRawXattrs(); + final boolean preserveAcls = context.shouldPreserve(FileAttribute.ACL); + final boolean preserveXAttrs = context.shouldPreserve(FileAttribute.XATTR); + final boolean preserveRawXattrs = context.shouldPreserveRawXattrs(); assert numListstatusThreads > 0; if (LOG.isDebugEnabled()) { @@ -649,7 +658,7 @@ public class SimpleCopyListing extends CopyListing { preserveAcls && child.isDirectory(), preserveXAttrs && child.isDirectory(), preserveRawXattrs && child.isDirectory(), - options.getBlocksPerChunk()); + context.getBlocksPerChunk()); for (CopyListingFileStatus fs : childCopyListingStatus) { if (randomizeFileListing) { @@ -681,9 +690,9 @@ public class SimpleCopyListing extends CopyListing { private void writeToFileListingRoot(SequenceFile.Writer fileListWriter, LinkedList fileStatus, Path sourcePathRoot, - DistCpOptions options) throws IOException { - boolean syncOrOverwrite = options.shouldSyncFolder() || - options.shouldOverwrite(); + DistCpContext context) throws IOException { + boolean syncOrOverwrite = context.shouldSyncFolder() || + context.shouldOverwrite(); for (CopyListingFileStatus fs : fileStatus) { if (fs.getPath().equals(sourcePathRoot) && fs.isDirectory() && syncOrOverwrite) { diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java index 6ddaab99c3d..81c2be7e05d 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java @@ -36,6 +36,7 @@ import org.apache.hadoop.tools.CopyListing; import org.apache.hadoop.tools.CopyListingFileStatus; import org.apache.hadoop.tools.DistCpConstants; import org.apache.hadoop.tools.DistCpOptionSwitch; +import org.apache.hadoop.tools.DistCpContext; import org.apache.hadoop.tools.DistCpOptions; import org.apache.hadoop.tools.DistCpOptions.FileAttribute; import org.apache.hadoop.tools.GlobbedCopyListing; @@ -354,16 +355,18 @@ public class CopyCommitter extends FileOutputCommitter { Path resultNonePath = Path.getPathWithoutSchemeAndAuthority(targetFinalPath) .toString().startsWith(DistCpConstants.HDFS_RESERVED_RAW_DIRECTORY_NAME) ? DistCpConstants.RAW_NONE_PATH : DistCpConstants.NONE_PATH; - DistCpOptions options = new DistCpOptions(targets, resultNonePath); // // Set up options to be the same from the CopyListing.buildListing's perspective, // so to collect similar listings as when doing the copy // - options.setOverwrite(overwrite); - options.setSyncFolder(syncFolder); - options.setTargetPathExists(targetPathExists); - - target.buildListing(targetListing, options); + DistCpOptions options = new DistCpOptions.Builder(targets, resultNonePath) + .withOverwrite(overwrite) + .withSyncFolder(syncFolder) + .build(); + DistCpContext distCpContext = new DistCpContext(options); + distCpContext.setTargetPathExists(targetPathExists); + + target.buildListing(targetListing, distCpContext); Path sortedTargetListing = DistCpUtils.sortListing(clusterFS, conf, targetListing); long totalLen = clusterFS.getFileStatus(sortedTargetListing).getLen(); diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java index e315b848de4..dbe750a65c4 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java @@ -39,7 +39,7 @@ import org.apache.hadoop.mapreduce.InputFormat; import org.apache.hadoop.tools.CopyListing.AclsNotSupportedException; import org.apache.hadoop.tools.CopyListing.XAttrsNotSupportedException; import org.apache.hadoop.tools.CopyListingFileStatus; -import org.apache.hadoop.tools.DistCpOptions; +import org.apache.hadoop.tools.DistCpContext; import org.apache.hadoop.tools.DistCpOptions.FileAttribute; import org.apache.hadoop.tools.mapred.UniformSizeInputFormat; import org.apache.hadoop.util.StringUtils; @@ -116,13 +116,13 @@ public class DistCpUtils { * a particular strategy from distcp-default.xml * * @param conf - Configuration object - * @param options - Handle to input options + * @param context - Distcp context with associated input options * @return Class implementing the strategy specified in options. */ public static Class getStrategy(Configuration conf, - DistCpOptions options) { + DistCpContext context) { String confLabel = "distcp." - + StringUtils.toLowerCase(options.getCopyStrategy()) + + StringUtils.toLowerCase(context.getCopyStrategy()) + ".strategy" + ".impl"; return conf.getClass(confLabel, UniformSizeInputFormat.class, InputFormat.class); } diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java index ea63e235138..97a6f62444b 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java @@ -103,20 +103,19 @@ public class TestCopyListing extends SimpleCopyListing { List srcPaths = new ArrayList(); srcPaths.add(new Path("/tmp/in/1")); srcPaths.add(new Path("/tmp/in/2")); - Path target = new Path("/tmp/out/1"); + final Path target = new Path("/tmp/out/1"); TestDistCpUtils.createFile(fs, "/tmp/in/1"); TestDistCpUtils.createFile(fs, "/tmp/in/2"); fs.mkdirs(target); - DistCpOptions options = new DistCpOptions(srcPaths, target); - validatePaths(options); + final DistCpOptions options = new DistCpOptions.Builder(srcPaths, target) + .build(); + validatePaths(new DistCpContext(options)); TestDistCpUtils.delete(fs, "/tmp"); //No errors - target = new Path("/tmp/out/1"); fs.create(target).close(); - options = new DistCpOptions(srcPaths, target); try { - validatePaths(options); + validatePaths(new DistCpContext(options)); Assert.fail("Invalid inputs accepted"); } catch (InvalidInputException ignore) { } TestDistCpUtils.delete(fs, "/tmp"); @@ -124,11 +123,9 @@ public class TestCopyListing extends SimpleCopyListing { srcPaths.clear(); srcPaths.add(new Path("/tmp/in/1")); fs.mkdirs(new Path("/tmp/in/1")); - target = new Path("/tmp/out/1"); fs.create(target).close(); - options = new DistCpOptions(srcPaths, target); try { - validatePaths(options); + validatePaths(new DistCpContext(options)); Assert.fail("Invalid inputs accepted"); } catch (InvalidInputException ignore) { } TestDistCpUtils.delete(fs, "/tmp"); @@ -151,10 +148,13 @@ public class TestCopyListing extends SimpleCopyListing { TestDistCpUtils.createFile(fs, "/tmp/in/src2/1.txt"); Path target = new Path("/tmp/out"); Path listingFile = new Path("/tmp/list"); - DistCpOptions options = new DistCpOptions(srcPaths, target); - CopyListing listing = CopyListing.getCopyListing(getConf(), CREDENTIALS, options); + final DistCpOptions options = new DistCpOptions.Builder(srcPaths, target) + .build(); + final DistCpContext context = new DistCpContext(options); + CopyListing listing = CopyListing.getCopyListing(getConf(), CREDENTIALS, + context); try { - listing.buildListing(listingFile, options); + listing.buildListing(listingFile, context); Assert.fail("Duplicates not detected"); } catch (DuplicateFileException ignore) { } @@ -196,11 +196,12 @@ public class TestCopyListing extends SimpleCopyListing { Path listingFile = new Path("/tmp/file"); - DistCpOptions options = new DistCpOptions(srcPaths, target); - options.setSyncFolder(true); + final DistCpOptions options = new DistCpOptions.Builder(srcPaths, target) + .withSyncFolder(true) + .build(); CopyListing listing = new SimpleCopyListing(getConf(), CREDENTIALS); try { - listing.buildListing(listingFile, options); + listing.buildListing(listingFile, new DistCpContext(options)); Assert.fail("Duplicates not detected"); } catch (DuplicateFileException ignore) { } @@ -209,7 +210,7 @@ public class TestCopyListing extends SimpleCopyListing { TestDistCpUtils.delete(fs, "/tmp"); try { - listing.buildListing(listingFile, options); + listing.buildListing(listingFile, new DistCpContext(options)); Assert.fail("Invalid input not detected"); } catch (InvalidInputException ignore) { } @@ -244,14 +245,14 @@ public class TestCopyListing extends SimpleCopyListing { } Path listingFile = new Path("/tmp/file"); - DistCpOptions options = new DistCpOptions(srcPaths, target); - options.setSyncFolder(true); + final DistCpOptions options = new DistCpOptions.Builder(srcPaths, target) + .withSyncFolder(true).build(); // Check without randomizing files getConf().setBoolean( DistCpConstants.CONF_LABEL_SIMPLE_LISTING_RANDOMIZE_FILES, false); SimpleCopyListing listing = new SimpleCopyListing(getConf(), CREDENTIALS); - listing.buildListing(listingFile, options); + listing.buildListing(listingFile, new DistCpContext(options)); Assert.assertEquals(listing.getNumberOfPaths(), pathCount); validateFinalListing(listingFile, srcFiles); @@ -265,7 +266,7 @@ public class TestCopyListing extends SimpleCopyListing { // Set the seed for randomness, so that it can be verified later long seed = System.nanoTime(); listing.setSeedForRandomListing(seed); - listing.buildListing(listingFile, options); + listing.buildListing(listingFile, new DistCpContext(options)); Assert.assertEquals(listing.getNumberOfPaths(), pathCount); // validate randomness @@ -322,11 +323,12 @@ public class TestCopyListing extends SimpleCopyListing { List srcPaths = new ArrayList(); srcPaths.add(sourceFile); - DistCpOptions options = new DistCpOptions(srcPaths, targetFile); + DistCpOptions options = new DistCpOptions.Builder(srcPaths, targetFile) + .build(); CopyListing listing = new SimpleCopyListing(getConf(), CREDENTIALS); final Path listFile = new Path(testRoot, "/tmp/fileList.seq"); - listing.buildListing(listFile, options); + listing.buildListing(listFile, new DistCpContext(options)); reader = new SequenceFile.Reader(getConf(), SequenceFile.Reader.file(listFile)); @@ -359,10 +361,11 @@ public class TestCopyListing extends SimpleCopyListing { doThrow(expectedEx).when(writer).close(); SimpleCopyListing listing = new SimpleCopyListing(getConf(), CREDENTIALS); - DistCpOptions options = new DistCpOptions(srcs, new Path(outFile.toURI())); + final DistCpOptions options = new DistCpOptions.Builder(srcs, + new Path(outFile.toURI())).build(); Exception actualEx = null; try { - listing.doBuildListing(writer, options); + listing.doBuildListing(writer, new DistCpContext(options)); } catch (Exception e) { actualEx = e; } diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpOptions.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpOptions.java new file mode 100644 index 00000000000..35251943c02 --- /dev/null +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpOptions.java @@ -0,0 +1,500 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.tools; + +import java.util.Collections; + +import org.junit.Assert; +import org.junit.Test; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.tools.DistCpOptions.FileAttribute; + +import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains; +import static org.apache.hadoop.tools.DistCpOptions.MAX_NUM_LISTSTATUS_THREADS; +import static org.junit.Assert.fail; + +/** + * This is to test constructing {@link DistCpOptions} manually with setters. + * + * The test cases in this class is very similar to the parser test, see + * {@link TestOptionsParser}. + */ +public class TestDistCpOptions { + + private static final float DELTA = 0.001f; + + @Test + public void testSetIgnoreFailure() { + final DistCpOptions.Builder builder = new DistCpOptions.Builder( + Collections.singletonList(new Path("hdfs://localhost:8020/source")), + new Path("hdfs://localhost:8020/target/")); + Assert.assertFalse(builder.build().shouldIgnoreFailures()); + + builder.withIgnoreFailures(true); + Assert.assertTrue(builder.build().shouldIgnoreFailures()); + } + + @Test + public void testSetOverwrite() { + final DistCpOptions.Builder builder = new DistCpOptions.Builder( + Collections.singletonList(new Path("hdfs://localhost:8020/source")), + new Path("hdfs://localhost:8020/target/")); + Assert.assertFalse(builder.build().shouldOverwrite()); + + builder.withOverwrite(true); + Assert.assertTrue(builder.build().shouldOverwrite()); + + try { + builder.withSyncFolder(true).build(); + Assert.fail("Update and overwrite aren't allowed together"); + } catch (IllegalArgumentException ignore) { + } + } + + @Test + public void testLogPath() { + final DistCpOptions.Builder builder = new DistCpOptions.Builder( + Collections.singletonList(new Path("hdfs://localhost:8020/source")), + new Path("hdfs://localhost:8020/target/")); + Assert.assertNull(builder.build().getLogPath()); + + final Path logPath = new Path("hdfs://localhost:8020/logs"); + builder.withLogPath(logPath); + Assert.assertEquals(logPath, builder.build().getLogPath()); + } + + @Test + public void testSetBlokcing() { + final DistCpOptions.Builder builder = new DistCpOptions.Builder( + Collections.singletonList(new Path("hdfs://localhost:8020/source")), + new Path("hdfs://localhost:8020/target/")); + Assert.assertTrue(builder.build().shouldBlock()); + + builder.withBlocking(false); + Assert.assertFalse(builder.build().shouldBlock()); + } + + @Test + public void testSetBandwidth() { + final DistCpOptions.Builder builder = new DistCpOptions.Builder( + Collections.singletonList(new Path("hdfs://localhost:8020/source")), + new Path("hdfs://localhost:8020/target/")); + Assert.assertEquals(0, builder.build().getMapBandwidth(), DELTA); + + builder.withMapBandwidth(11); + Assert.assertEquals(11, builder.build().getMapBandwidth(), DELTA); + } + + @Test(expected = IllegalArgumentException.class) + public void testSetNonPositiveBandwidth() { + new DistCpOptions.Builder( + Collections.singletonList(new Path("hdfs://localhost:8020/source")), + new Path("hdfs://localhost:8020/target/")) + .withMapBandwidth(-11) + .build(); + } + + @Test(expected = IllegalArgumentException.class) + public void testSetZeroBandwidth() { + new DistCpOptions.Builder( + Collections.singletonList(new Path("hdfs://localhost:8020/source")), + new Path("hdfs://localhost:8020/target/")) + .withMapBandwidth(0) + .build(); + } + + @Test + public void testSetSkipCRC() { + final DistCpOptions.Builder builder = new DistCpOptions.Builder( + Collections.singletonList(new Path("hdfs://localhost:8020/source")), + new Path("hdfs://localhost:8020/target/")); + Assert.assertFalse(builder.build().shouldSkipCRC()); + + final DistCpOptions options = builder.withSyncFolder(true).withCRC(true) + .build(); + Assert.assertTrue(options.shouldSyncFolder()); + Assert.assertTrue(options.shouldSkipCRC()); + } + + @Test + public void testSetAtomicCommit() { + final DistCpOptions.Builder builder = new DistCpOptions.Builder( + Collections.singletonList(new Path("hdfs://localhost:8020/source")), + new Path("hdfs://localhost:8020/target/")); + Assert.assertFalse(builder.build().shouldAtomicCommit()); + + builder.withAtomicCommit(true); + Assert.assertTrue(builder.build().shouldAtomicCommit()); + + try { + builder.withSyncFolder(true).build(); + Assert.fail("Atomic and sync folders were mutually exclusive"); + } catch (IllegalArgumentException ignore) { + } + } + + @Test + public void testSetWorkPath() { + final DistCpOptions.Builder builder = new DistCpOptions.Builder( + Collections.singletonList(new Path("hdfs://localhost:8020/source")), + new Path("hdfs://localhost:8020/target/")); + Assert.assertNull(builder.build().getAtomicWorkPath()); + + builder.withAtomicCommit(true); + Assert.assertNull(builder.build().getAtomicWorkPath()); + + final Path workPath = new Path("hdfs://localhost:8020/work"); + builder.withAtomicWorkPath(workPath); + Assert.assertEquals(workPath, builder.build().getAtomicWorkPath()); + } + + @Test + public void testSetSyncFolders() { + final DistCpOptions.Builder builder = new DistCpOptions.Builder( + Collections.singletonList(new Path("hdfs://localhost:8020/source")), + new Path("hdfs://localhost:8020/target/")); + Assert.assertFalse(builder.build().shouldSyncFolder()); + + builder.withSyncFolder(true); + Assert.assertTrue(builder.build().shouldSyncFolder()); + } + + @Test + public void testSetDeleteMissing() { + final DistCpOptions.Builder builder = new DistCpOptions.Builder( + Collections.singletonList(new Path("hdfs://localhost:8020/source")), + new Path("hdfs://localhost:8020/target/")); + Assert.assertFalse(builder.build().shouldDeleteMissing()); + + DistCpOptions options = builder.withSyncFolder(true) + .withDeleteMissing(true) + .build(); + Assert.assertTrue(options.shouldSyncFolder()); + Assert.assertTrue(options.shouldDeleteMissing()); + + options = new DistCpOptions.Builder( + Collections.singletonList(new Path("hdfs://localhost:8020/source")), + new Path("hdfs://localhost:8020/target/")) + .withOverwrite(true) + .withDeleteMissing(true) + .build(); + Assert.assertTrue(options.shouldOverwrite()); + Assert.assertTrue(options.shouldDeleteMissing()); + + try { + new DistCpOptions.Builder( + Collections.singletonList(new Path("hdfs://localhost:8020/source")), + new Path("hdfs://localhost:8020/target/")) + .withDeleteMissing(true) + .build(); + fail("Delete missing should fail without update or overwrite options"); + } catch (IllegalArgumentException e) { + assertExceptionContains("Delete missing is applicable only with update " + + "or overwrite options", e); + } + try { + new DistCpOptions.Builder( + new Path("hdfs://localhost:8020/source/first"), + new Path("hdfs://localhost:8020/target/")) + .withSyncFolder(true) + .withDeleteMissing(true) + .withUseDiff("s1", "s2") + .build(); + fail("Should have failed as -delete and -diff are mutually exclusive."); + } catch (IllegalArgumentException e) { + assertExceptionContains( + "-delete and -diff/-rdiff are mutually exclusive.", e); + } + } + + @Test + public void testSetMaps() { + final DistCpOptions.Builder builder = new DistCpOptions.Builder( + Collections.singletonList(new Path("hdfs://localhost:8020/source")), + new Path("hdfs://localhost:8020/target/")); + Assert.assertEquals(DistCpConstants.DEFAULT_MAPS, + builder.build().getMaxMaps()); + + builder.maxMaps(1); + Assert.assertEquals(1, builder.build().getMaxMaps()); + + builder.maxMaps(0); + Assert.assertEquals(1, builder.build().getMaxMaps()); + } + + @Test + public void testSetNumListtatusThreads() { + final DistCpOptions.Builder builder = new DistCpOptions.Builder( + new Path("hdfs://localhost:8020/source/first"), + new Path("hdfs://localhost:8020/target/")); + // If command line argument isn't set, we expect .getNumListstatusThreads + // option to be zero (so that we know when to override conf properties). + Assert.assertEquals(0, builder.build().getNumListstatusThreads()); + + builder.withNumListstatusThreads(12); + Assert.assertEquals(12, builder.build().getNumListstatusThreads()); + + builder.withNumListstatusThreads(0); + Assert.assertEquals(0, builder.build().getNumListstatusThreads()); + + // Ignore large number of threads. + builder.withNumListstatusThreads(MAX_NUM_LISTSTATUS_THREADS * 2); + Assert.assertEquals(MAX_NUM_LISTSTATUS_THREADS, + builder.build().getNumListstatusThreads()); + } + + @Test + public void testSourceListing() { + final DistCpOptions.Builder builder = new DistCpOptions.Builder( + new Path("hdfs://localhost:8020/source/first"), + new Path("hdfs://localhost:8020/target/")); + Assert.assertEquals(new Path("hdfs://localhost:8020/source/first"), + builder.build().getSourceFileListing()); + } + + @Test(expected = IllegalArgumentException.class) + public void testMissingTarget() { + new DistCpOptions.Builder(new Path("hdfs://localhost:8020/source/first"), + null); + } + + @Test + public void testToString() { + DistCpOptions option = new DistCpOptions.Builder(new Path("abc"), + new Path("xyz")).build(); + String val = "DistCpOptions{atomicCommit=false, syncFolder=false, " + + "deleteMissing=false, ignoreFailures=false, overwrite=false, " + + "append=false, useDiff=false, useRdiff=false, " + + "fromSnapshot=null, toSnapshot=null, " + + "skipCRC=false, blocking=true, numListstatusThreads=0, maxMaps=20, " + + "mapBandwidth=0.0, copyStrategy='uniformsize', preserveStatus=[], " + + "atomicWorkPath=null, logPath=null, sourceFileListing=abc, " + + "sourcePaths=null, targetPath=xyz, filtersFile='null'," + + " blocksPerChunk=0}"; + String optionString = option.toString(); + Assert.assertEquals(val, optionString); + Assert.assertNotSame(DistCpOptionSwitch.ATOMIC_COMMIT.toString(), + DistCpOptionSwitch.ATOMIC_COMMIT.name()); + } + + @Test + public void testCopyStrategy() { + final DistCpOptions.Builder builder = new DistCpOptions.Builder( + new Path("hdfs://localhost:8020/source/first"), + new Path("hdfs://localhost:8020/target/")); + Assert.assertEquals(DistCpConstants.UNIFORMSIZE, + builder.build().getCopyStrategy()); + builder.withCopyStrategy("dynamic"); + Assert.assertEquals("dynamic", builder.build().getCopyStrategy()); + } + + @Test + public void testTargetPath() { + final DistCpOptions options = new DistCpOptions.Builder( + new Path("hdfs://localhost:8020/source/first"), + new Path("hdfs://localhost:8020/target/")).build(); + Assert.assertEquals(new Path("hdfs://localhost:8020/target/"), + options.getTargetPath()); + } + + @Test + public void testPreserve() { + DistCpOptions options = new DistCpOptions.Builder( + new Path("hdfs://localhost:8020/source/first"), + new Path("hdfs://localhost:8020/target/")) + .build(); + Assert.assertFalse(options.shouldPreserve(FileAttribute.BLOCKSIZE)); + Assert.assertFalse(options.shouldPreserve(FileAttribute.REPLICATION)); + Assert.assertFalse(options.shouldPreserve(FileAttribute.PERMISSION)); + Assert.assertFalse(options.shouldPreserve(FileAttribute.USER)); + Assert.assertFalse(options.shouldPreserve(FileAttribute.GROUP)); + Assert.assertFalse(options.shouldPreserve(FileAttribute.CHECKSUMTYPE)); + + options = new DistCpOptions.Builder( + new Path("hdfs://localhost:8020/source/first"), + new Path("hdfs://localhost:8020/target/")) + .preserve(FileAttribute.ACL) + .build(); + Assert.assertFalse(options.shouldPreserve(FileAttribute.BLOCKSIZE)); + Assert.assertFalse(options.shouldPreserve(FileAttribute.REPLICATION)); + Assert.assertFalse(options.shouldPreserve(FileAttribute.PERMISSION)); + Assert.assertFalse(options.shouldPreserve(FileAttribute.USER)); + Assert.assertFalse(options.shouldPreserve(FileAttribute.GROUP)); + Assert.assertFalse(options.shouldPreserve(FileAttribute.CHECKSUMTYPE)); + Assert.assertTrue(options.shouldPreserve(FileAttribute.ACL)); + + options = new DistCpOptions.Builder( + new Path("hdfs://localhost:8020/source/first"), + new Path("hdfs://localhost:8020/target/")) + .preserve(FileAttribute.BLOCKSIZE) + .preserve(FileAttribute.REPLICATION) + .preserve(FileAttribute.PERMISSION) + .preserve(FileAttribute.USER) + .preserve(FileAttribute.GROUP) + .preserve(FileAttribute.CHECKSUMTYPE) + .build(); + + Assert.assertTrue(options.shouldPreserve(FileAttribute.BLOCKSIZE)); + Assert.assertTrue(options.shouldPreserve(FileAttribute.REPLICATION)); + Assert.assertTrue(options.shouldPreserve(FileAttribute.PERMISSION)); + Assert.assertTrue(options.shouldPreserve(FileAttribute.USER)); + Assert.assertTrue(options.shouldPreserve(FileAttribute.GROUP)); + Assert.assertTrue(options.shouldPreserve(FileAttribute.CHECKSUMTYPE)); + Assert.assertFalse(options.shouldPreserve(FileAttribute.XATTR)); + } + + @Test + public void testAppendOption() { + final DistCpOptions.Builder builder = new DistCpOptions.Builder( + Collections.singletonList(new Path("hdfs://localhost:8020/source")), + new Path("hdfs://localhost:8020/target/")) + .withSyncFolder(true) + .withAppend(true); + Assert.assertTrue(builder.build().shouldAppend()); + + try { + // make sure -append is only valid when -update is specified + new DistCpOptions.Builder( + Collections.singletonList(new Path("hdfs://localhost:8020/source")), + new Path("hdfs://localhost:8020/target/")) + .withAppend(true) + .build(); + fail("Append should fail if update option is not specified"); + } catch (IllegalArgumentException e) { + assertExceptionContains( + "Append is valid only with update options", e); + } + + try { + // make sure -append is invalid when skipCrc is specified + new DistCpOptions.Builder( + Collections.singletonList(new Path("hdfs://localhost:8020/source")), + new Path("hdfs://localhost:8020/target/")) + .withSyncFolder(true) + .withAppend(true) + .withCRC(true) + .build(); + fail("Append should fail if skipCrc option is specified"); + } catch (IllegalArgumentException e) { + assertExceptionContains( + "Append is disallowed when skipping CRC", e); + } + } + + @Test + public void testDiffOption() { + DistCpOptions options = new DistCpOptions.Builder( + new Path("hdfs://localhost:8020/source/first"), + new Path("hdfs://localhost:8020/target/")) + .withSyncFolder(true) + .withUseDiff("s1", "s2") + .build(); + Assert.assertTrue(options.shouldUseDiff()); + Assert.assertEquals("s1", options.getFromSnapshot()); + Assert.assertEquals("s2", options.getToSnapshot()); + + options = new DistCpOptions.Builder( + new Path("hdfs://localhost:8020/source/first"), + new Path("hdfs://localhost:8020/target/")) + .withSyncFolder(true) + .withUseDiff("s1", ".") + .build(); + Assert.assertTrue(options.shouldUseDiff()); + Assert.assertEquals("s1", options.getFromSnapshot()); + Assert.assertEquals(".", options.getToSnapshot()); + + // make sure -diff is only valid when -update is specified + try { + new DistCpOptions.Builder(new Path("hdfs://localhost:8020/source/first"), + new Path("hdfs://localhost:8020/target/")) + .withUseDiff("s1", "s2") + .build(); + fail("-diff should fail if -update option is not specified"); + } catch (IllegalArgumentException e) { + assertExceptionContains( + "-diff/-rdiff is valid only with -update option", e); + } + + try { + new DistCpOptions.Builder( + new Path("hdfs://localhost:8020/source/first"), + new Path("hdfs://localhost:8020/target/")) + .withSyncFolder(true) + .withUseDiff("s1", "s2") + .withDeleteMissing(true) + .build(); + fail("Should fail as -delete and -diff/-rdiff are mutually exclusive."); + } catch (IllegalArgumentException e) { + assertExceptionContains( + "-delete and -diff/-rdiff are mutually exclusive.", e); + } + + try { + new DistCpOptions.Builder(new Path("hdfs://localhost:8020/source/first"), + new Path("hdfs://localhost:8020/target/")) + .withUseDiff("s1", "s2") + .withDeleteMissing(true) + .build(); + fail("-diff should fail if -update option is not specified"); + } catch (IllegalArgumentException e) { + assertExceptionContains( + "-delete and -diff/-rdiff are mutually exclusive.", e); + } + + try { + new DistCpOptions.Builder(new Path("hdfs://localhost:8020/source/first"), + new Path("hdfs://localhost:8020/target/")) + .withDeleteMissing(true) + .withUseDiff("s1", "s2") + .build(); + fail("Should have failed as -delete and -diff are mutually exclusive"); + } catch (IllegalArgumentException e) { + assertExceptionContains( + "-delete and -diff/-rdiff are mutually exclusive", e); + } + } + + @Test + public void testExclusionsOption() { + final DistCpOptions.Builder builder = new DistCpOptions.Builder( + new Path("hdfs://localhost:8020/source/first"), + new Path("hdfs://localhost:8020/target/")); + Assert.assertNull(builder.build().getFiltersFile()); + + builder.withFiltersFile("/tmp/filters.txt"); + Assert.assertEquals("/tmp/filters.txt", builder.build().getFiltersFile()); + } + + @Test + public void testSetOptionsForSplitLargeFile() { + final DistCpOptions.Builder builder = new DistCpOptions.Builder( + new Path("hdfs://localhost:8020/source/"), + new Path("hdfs://localhost:8020/target/")) + .withAppend(true) + .withSyncFolder(true); + Assert.assertFalse(builder.build().shouldPreserve(FileAttribute.BLOCKSIZE)); + Assert.assertTrue(builder.build().shouldAppend()); + + builder.withBlocksPerChunk(5440); + Assert.assertTrue(builder.build().shouldPreserve(FileAttribute.BLOCKSIZE)); + Assert.assertFalse(builder.build().shouldAppend()); + } + +} diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java index 94e860412d2..717b2f08436 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java @@ -39,7 +39,7 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -47,7 +47,7 @@ public class TestDistCpSync { private MiniDFSCluster cluster; private final Configuration conf = new HdfsConfiguration(); private DistributedFileSystem dfs; - private DistCpOptions options; + private DistCpContext context; private final Path source = new Path("/source"); private final Path target = new Path("/target"); private final long BLOCK_SIZE = 1024; @@ -62,10 +62,13 @@ public class TestDistCpSync { dfs.mkdirs(source); dfs.mkdirs(target); - options = new DistCpOptions(Arrays.asList(source), target); - options.setSyncFolder(true); - options.setUseDiff("s1", "s2"); + final DistCpOptions options = new DistCpOptions.Builder( + Collections.singletonList(source), target) + .withSyncFolder(true) + .withUseDiff("s1", "s2") + .build(); options.appendToConf(conf); + context = new DistCpContext(options); conf.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, target.toString()); conf.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, target.toString()); @@ -92,34 +95,34 @@ public class TestDistCpSync { // make sure the source path has been updated to the snapshot path final Path spath = new Path(source, HdfsConstants.DOT_SNAPSHOT_DIR + Path.SEPARATOR + "s2"); - Assert.assertEquals(spath, options.getSourcePaths().get(0)); + Assert.assertEquals(spath, context.getSourcePaths().get(0)); // reset source path in options - options.setSourcePaths(Arrays.asList(source)); + context.setSourcePaths(Collections.singletonList(source)); // the source/target does not have the given snapshots dfs.allowSnapshot(source); dfs.allowSnapshot(target); Assert.assertFalse(sync()); - Assert.assertEquals(spath, options.getSourcePaths().get(0)); + Assert.assertEquals(spath, context.getSourcePaths().get(0)); // reset source path in options - options.setSourcePaths(Arrays.asList(source)); + context.setSourcePaths(Collections.singletonList(source)); dfs.createSnapshot(source, "s1"); dfs.createSnapshot(source, "s2"); dfs.createSnapshot(target, "s1"); Assert.assertTrue(sync()); // reset source paths in options - options.setSourcePaths(Arrays.asList(source)); + context.setSourcePaths(Collections.singletonList(source)); // changes have been made in target final Path subTarget = new Path(target, "sub"); dfs.mkdirs(subTarget); Assert.assertFalse(sync()); // make sure the source path has been updated to the snapshot path - Assert.assertEquals(spath, options.getSourcePaths().get(0)); + Assert.assertEquals(spath, context.getSourcePaths().get(0)); // reset source paths in options - options.setSourcePaths(Arrays.asList(source)); + context.setSourcePaths(Collections.singletonList(source)); dfs.delete(subTarget, true); Assert.assertTrue(sync()); } @@ -137,7 +140,7 @@ public class TestDistCpSync { } private boolean sync() throws Exception { - DistCpSync distCpSync = new DistCpSync(options, conf); + DistCpSync distCpSync = new DistCpSync(context, conf); return distCpSync.sync(); } @@ -231,7 +234,7 @@ public class TestDistCpSync { SnapshotDiffReport report = dfs.getSnapshotDiffReport(source, "s1", "s2"); System.out.println(report); - DistCpSync distCpSync = new DistCpSync(options, conf); + DistCpSync distCpSync = new DistCpSync(context, conf); // do the sync Assert.assertTrue(distCpSync.sync()); @@ -239,24 +242,24 @@ public class TestDistCpSync { // make sure the source path has been updated to the snapshot path final Path spath = new Path(source, HdfsConstants.DOT_SNAPSHOT_DIR + Path.SEPARATOR + "s2"); - Assert.assertEquals(spath, options.getSourcePaths().get(0)); + Assert.assertEquals(spath, context.getSourcePaths().get(0)); // build copy listing final Path listingPath = new Path("/tmp/META/fileList.seq"); CopyListing listing = new SimpleCopyListing(conf, new Credentials(), distCpSync); - listing.buildListing(listingPath, options); + listing.buildListing(listingPath, context); Map copyListing = getListing(listingPath); CopyMapper copyMapper = new CopyMapper(); StubContext stubContext = new StubContext(conf, null, 0); - Mapper.Context context = + Mapper.Context mapContext = stubContext.getContext(); // Enable append - context.getConfiguration().setBoolean( + mapContext.getConfiguration().setBoolean( DistCpOptionSwitch.APPEND.getConfigLabel(), true); - copyMapper.setup(context); + copyMapper.setup(mapContext); for (Map.Entry entry : copyListing.entrySet()) { - copyMapper.map(entry.getKey(), entry.getValue(), context); + copyMapper.map(entry.getKey(), entry.getValue(), mapContext); } // verify that we only list modified and created files/directories @@ -312,7 +315,12 @@ public class TestDistCpSync { */ @Test public void testSyncWithCurrent() throws Exception { - options.setUseDiff("s1", "."); + final DistCpOptions options = new DistCpOptions.Builder( + Collections.singletonList(source), target) + .withSyncFolder(true) + .withUseDiff("s1", ".") + .build(); + context = new DistCpContext(options); initData(source); initData(target); enableAndCreateFirstSnapshot(); @@ -323,7 +331,7 @@ public class TestDistCpSync { // do the sync sync(); // make sure the source path is still unchanged - Assert.assertEquals(source, options.getSourcePaths().get(0)); + Assert.assertEquals(source, context.getSourcePaths().get(0)); } private void initData2(Path dir) throws Exception { @@ -501,32 +509,32 @@ public class TestDistCpSync { SnapshotDiffReport report = dfs.getSnapshotDiffReport(source, "s1", "s2"); System.out.println(report); - DistCpSync distCpSync = new DistCpSync(options, conf); + DistCpSync distCpSync = new DistCpSync(context, conf); // do the sync Assert.assertTrue(distCpSync.sync()); // make sure the source path has been updated to the snapshot path final Path spath = new Path(source, HdfsConstants.DOT_SNAPSHOT_DIR + Path.SEPARATOR + "s2"); - Assert.assertEquals(spath, options.getSourcePaths().get(0)); + Assert.assertEquals(spath, context.getSourcePaths().get(0)); // build copy listing final Path listingPath = new Path("/tmp/META/fileList.seq"); CopyListing listing = new SimpleCopyListing(conf, new Credentials(), distCpSync); - listing.buildListing(listingPath, options); + listing.buildListing(listingPath, context); Map copyListing = getListing(listingPath); CopyMapper copyMapper = new CopyMapper(); StubContext stubContext = new StubContext(conf, null, 0); - Mapper.Context context = + Mapper.Context mapContext = stubContext.getContext(); // Enable append - context.getConfiguration().setBoolean( + mapContext.getConfiguration().setBoolean( DistCpOptionSwitch.APPEND.getConfigLabel(), true); - copyMapper.setup(context); + copyMapper.setup(mapContext); for (Map.Entry entry : copyListing.entrySet()) { - copyMapper.map(entry.getKey(), entry.getValue(), context); + copyMapper.map(entry.getKey(), entry.getValue(), mapContext); } // verify that we only list modified and created files/directories @@ -729,7 +737,7 @@ public class TestDistCpSync { boolean threwException = false; try { - DistCpSync distCpSync = new DistCpSync(options, conf); + DistCpSync distCpSync = new DistCpSync(context, conf); // do the sync distCpSync.sync(); } catch (HadoopIllegalArgumentException e) { diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSyncReverseBase.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSyncReverseBase.java index fea374ee1a7..cca1c5381c4 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSyncReverseBase.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSyncReverseBase.java @@ -56,7 +56,8 @@ public abstract class TestDistCpSyncReverseBase { private MiniDFSCluster cluster; private final Configuration conf = new HdfsConfiguration(); private DistributedFileSystem dfs; - private DistCpOptions options; + private DistCpOptions.Builder optionsBuilder; + private DistCpContext distCpContext; private Path source; private boolean isSrcNotSameAsTgt = true; private final Path target = new Path("/target"); @@ -139,10 +140,12 @@ public abstract class TestDistCpSyncReverseBase { } dfs.mkdirs(target); - options = new DistCpOptions(Arrays.asList(source), target); - options.setSyncFolder(true); - options.setUseRdiff("s2", "s1"); + optionsBuilder = new DistCpOptions.Builder(Arrays.asList(source), target) + .withSyncFolder(true) + .withUseRdiff("s2", "s1"); + final DistCpOptions options = optionsBuilder.build(); options.appendToConf(conf); + distCpContext = new DistCpContext(options); conf.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, target.toString()); conf.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, target.toString()); @@ -169,33 +172,33 @@ public abstract class TestDistCpSyncReverseBase { // make sure the source path has been updated to the snapshot path final Path spath = new Path(source, HdfsConstants.DOT_SNAPSHOT_DIR + Path.SEPARATOR + "s1"); - Assert.assertEquals(spath, options.getSourcePaths().get(0)); + Assert.assertEquals(spath, distCpContext.getSourcePaths().get(0)); // reset source path in options - options.setSourcePaths(Arrays.asList(source)); + optionsBuilder.withSourcePaths(Arrays.asList(source)); // the source/target does not have the given snapshots dfs.allowSnapshot(source); dfs.allowSnapshot(target); Assert.assertFalse(sync()); - Assert.assertEquals(spath, options.getSourcePaths().get(0)); + Assert.assertEquals(spath, distCpContext.getSourcePaths().get(0)); // reset source path in options - options.setSourcePaths(Arrays.asList(source)); + optionsBuilder.withSourcePaths(Arrays.asList(source)); this.enableAndCreateFirstSnapshot(); dfs.createSnapshot(target, "s2"); Assert.assertTrue(sync()); // reset source paths in options - options.setSourcePaths(Arrays.asList(source)); + optionsBuilder.withSourcePaths(Arrays.asList(source)); // changes have been made in target final Path subTarget = new Path(target, "sub"); dfs.mkdirs(subTarget); Assert.assertFalse(sync()); // make sure the source path has been updated to the snapshot path - Assert.assertEquals(spath, options.getSourcePaths().get(0)); + Assert.assertEquals(spath, distCpContext.getSourcePaths().get(0)); // reset source paths in options - options.setSourcePaths(Arrays.asList(source)); + optionsBuilder.withSourcePaths(Arrays.asList(source)); dfs.delete(subTarget, true); Assert.assertTrue(sync()); } @@ -215,7 +218,8 @@ public abstract class TestDistCpSyncReverseBase { } private boolean sync() throws Exception { - DistCpSync distCpSync = new DistCpSync(options, conf); + distCpContext = new DistCpContext(optionsBuilder.build()); + final DistCpSync distCpSync = new DistCpSync(distCpContext, conf); return distCpSync.sync(); } @@ -328,7 +332,7 @@ public abstract class TestDistCpSyncReverseBase { SnapshotDiffReport report = dfs.getSnapshotDiffReport(target, "s2", "s1"); System.out.println(report); - DistCpSync distCpSync = new DistCpSync(options, conf); + final DistCpSync distCpSync = new DistCpSync(distCpContext, conf); lsr("Before sync target: ", shell, target); @@ -340,13 +344,13 @@ public abstract class TestDistCpSyncReverseBase { // make sure the source path has been updated to the snapshot path final Path spath = new Path(source, HdfsConstants.DOT_SNAPSHOT_DIR + Path.SEPARATOR + "s1"); - Assert.assertEquals(spath, options.getSourcePaths().get(0)); + Assert.assertEquals(spath, distCpContext.getSourcePaths().get(0)); // build copy listing final Path listingPath = new Path("/tmp/META/fileList.seq"); CopyListing listing = new SimpleCopyListing(conf, new Credentials(), distCpSync); - listing.buildListing(listingPath, options); + listing.buildListing(listingPath, distCpContext); Map copyListing = getListing(listingPath); CopyMapper copyMapper = new CopyMapper(); @@ -425,7 +429,7 @@ public abstract class TestDistCpSyncReverseBase { */ @Test public void testSyncWithCurrent() throws Exception { - options.setUseRdiff(".", "s1"); + optionsBuilder.withUseRdiff(".", "s1"); if (isSrcNotSameAsTgt) { initData(source); } @@ -440,7 +444,7 @@ public abstract class TestDistCpSyncReverseBase { final Path spath = new Path(source, HdfsConstants.DOT_SNAPSHOT_DIR + Path.SEPARATOR + "s1"); // make sure the source path is still unchanged - Assert.assertEquals(spath, options.getSourcePaths().get(0)); + Assert.assertEquals(spath, distCpContext.getSourcePaths().get(0)); } private void initData2(Path dir) throws Exception { @@ -649,7 +653,7 @@ public abstract class TestDistCpSyncReverseBase { lsrSource("Before sync source: ", shell, source); lsr("Before sync target: ", shell, target); - DistCpSync distCpSync = new DistCpSync(options, conf); + DistCpSync distCpSync = new DistCpSync(distCpContext, conf); // do the sync distCpSync.sync(); @@ -658,12 +662,12 @@ public abstract class TestDistCpSyncReverseBase { // make sure the source path has been updated to the snapshot path final Path spath = new Path(source, HdfsConstants.DOT_SNAPSHOT_DIR + Path.SEPARATOR + "s1"); - Assert.assertEquals(spath, options.getSourcePaths().get(0)); + Assert.assertEquals(spath, distCpContext.getSourcePaths().get(0)); // build copy listing final Path listingPath = new Path("/tmp/META/fileList.seq"); CopyListing listing = new SimpleCopyListing(conf, new Credentials(), distCpSync); - listing.buildListing(listingPath, options); + listing.buildListing(listingPath, distCpContext); Map copyListing = getListing(listingPath); CopyMapper copyMapper = new CopyMapper(); diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpViewFs.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpViewFs.java index 5511e094ced..d6d05421a57 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpViewFs.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpViewFs.java @@ -413,11 +413,13 @@ public class TestDistCpViewFs { private void runTest(Path listFile, Path target, boolean targetExists, boolean sync) throws IOException { - DistCpOptions options = new DistCpOptions(listFile, target); - options.setSyncFolder(sync); - options.setTargetPathExists(targetExists); + final DistCpOptions options = new DistCpOptions.Builder(listFile, target) + .withSyncFolder(sync) + .build(); try { - new DistCp(getConf(), options).execute(); + final DistCp distcp = new DistCp(getConf(), options); + distcp.context.setTargetPathExists(targetExists); + distcp.execute(); } catch (Exception e) { LOG.error("Exception encountered ", e); throw new IOException(e); diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestFileBasedCopyListing.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestFileBasedCopyListing.java index fe2c66870e6..203de1a2adf 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestFileBasedCopyListing.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestFileBasedCopyListing.java @@ -514,10 +514,11 @@ public class TestFileBasedCopyListing { private void runTest(Path listFile, Path target, boolean targetExists, boolean sync) throws IOException { CopyListing listing = new FileBasedCopyListing(config, CREDENTIALS); - DistCpOptions options = new DistCpOptions(listFile, target); - options.setSyncFolder(sync); - options.setTargetPathExists(targetExists); - listing.buildListing(listFile, options); + final DistCpOptions options = new DistCpOptions.Builder(listFile, target) + .withSyncFolder(sync).build(); + final DistCpContext context = new DistCpContext(options); + context.setTargetPathExists(targetExists); + listing.buildListing(listFile, context); } private void checkResult(Path listFile, int count) throws IOException { diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestGlobbedCopyListing.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestGlobbedCopyListing.java index 6c03b4ee8a8..1c92a9c5ef2 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestGlobbedCopyListing.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestGlobbedCopyListing.java @@ -34,7 +34,7 @@ import org.junit.Test; import java.io.DataOutputStream; import java.net.URI; -import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -109,9 +109,12 @@ public class TestGlobbedCopyListing { Path source = new Path(fileSystemPath.toString() + "/tmp/source"); Path target = new Path(fileSystemPath.toString() + "/tmp/target"); Path listingPath = new Path(fileSystemPath.toString() + "/tmp/META/fileList.seq"); - DistCpOptions options = new DistCpOptions(Arrays.asList(source), target); - options.setTargetPathExists(false); - new GlobbedCopyListing(new Configuration(), CREDENTIALS).buildListing(listingPath, options); + DistCpOptions options = new DistCpOptions.Builder( + Collections.singletonList(source), target).build(); + DistCpContext context = new DistCpContext(options); + context.setTargetPathExists(false); + new GlobbedCopyListing(new Configuration(), CREDENTIALS) + .buildListing(listingPath, context); verifyContents(listingPath); } diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestIntegration.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestIntegration.java index ee8e7cc4f10..7574dedea2e 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestIntegration.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestIntegration.java @@ -26,7 +26,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapreduce.Cluster; import org.apache.hadoop.mapreduce.JobSubmissionFiles; -import org.apache.hadoop.security.Credentials; import org.apache.hadoop.tools.util.TestDistCpUtils; import org.junit.Assert; import org.junit.BeforeClass; @@ -493,7 +492,8 @@ public class TestIntegration { List sources = new ArrayList(); sources.add(sourcePath); - DistCpOptions options = new DistCpOptions(sources, target); + DistCpOptions options = new DistCpOptions.Builder(sources, target) + .build(); Configuration conf = getConf(); Path stagingDir = JobSubmissionFiles.getStagingDir( @@ -559,14 +559,16 @@ public class TestIntegration { private void runTest(Path listFile, Path target, boolean targetExists, boolean sync, boolean delete, boolean overwrite) throws IOException { - DistCpOptions options = new DistCpOptions(listFile, target); - options.setSyncFolder(sync); - options.setDeleteMissing(delete); - options.setOverwrite(overwrite); - options.setTargetPathExists(targetExists); - options.setNumListstatusThreads(numListstatusThreads); + final DistCpOptions options = new DistCpOptions.Builder(listFile, target) + .withSyncFolder(sync) + .withDeleteMissing(delete) + .withOverwrite(overwrite) + .withNumListstatusThreads(numListstatusThreads) + .build(); try { - new DistCp(getConf(), options).execute(); + final DistCp distCp = new DistCp(getConf(), options); + distCp.context.setTargetPathExists(targetExists); + distCp.execute(); } catch (Exception e) { LOG.error("Exception encountered ", e); throw new IOException(e); diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java index f94ba97ec34..e7fdc515a7f 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java @@ -18,7 +18,7 @@ package org.apache.hadoop.tools; -import static org.junit.Assert.assertFalse; +import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains; import static org.junit.Assert.fail; import org.junit.Assert; @@ -28,7 +28,6 @@ import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.tools.DistCpOptions.*; import org.apache.hadoop.conf.Configuration; -import java.util.Iterator; import java.util.NoSuchElementException; public class TestOptionsParser { @@ -329,7 +328,7 @@ public class TestOptionsParser { "100", "hdfs://localhost:9820/source/first", "hdfs://localhost:9820/target/"}); - Assert.assertEquals(DistCpOptions.maxNumListstatusThreads, + Assert.assertEquals(DistCpOptions.MAX_NUM_LISTSTATUS_THREADS, options.getNumListstatusThreads()); } @@ -382,25 +381,6 @@ public class TestOptionsParser { } catch (IllegalArgumentException ignore) {} } - @Test - public void testToString() { - DistCpOptions option = new DistCpOptions(new Path("abc"), new Path("xyz")); - String val = "DistCpOptions{atomicCommit=false, syncFolder=false, " - + "deleteMissing=false, ignoreFailures=false, overwrite=false, " - + "append=false, useDiff=false, useRdiff=false, " - + "fromSnapshot=null, toSnapshot=null, " - + "skipCRC=false, blocking=true, numListstatusThreads=0, maxMaps=20, " - + "mapBandwidth=0.0, " - + "copyStrategy='uniformsize', preserveStatus=[], " - + "preserveRawXattrs=false, atomicWorkPath=null, logPath=null, " - + "sourceFileListing=abc, sourcePaths=null, targetPath=xyz, " - + "targetPathExists=true, filtersFile='null', blocksPerChunk=0}"; - String optionString = option.toString(); - Assert.assertEquals(val, optionString); - Assert.assertNotSame(DistCpOptionSwitch.ATOMIC_COMMIT.toString(), - DistCpOptionSwitch.ATOMIC_COMMIT.name()); - } - @Test public void testCopyStrategy() { DistCpOptions options = OptionsParser.parse(new String[] { @@ -529,13 +509,8 @@ public class TestOptionsParser { "-f", "hdfs://localhost:9820/source/first", "hdfs://localhost:9820/target/"}); - int i = 0; - Iterator attribIterator = options.preserveAttributes(); - while (attribIterator.hasNext()) { - attribIterator.next(); - i++; - } - Assert.assertEquals(i, DistCpOptionSwitch.PRESERVE_STATUS_DEFAULT.length() - 2); + Assert.assertEquals(DistCpOptionSwitch.PRESERVE_STATUS_DEFAULT.length() - 2, + options.getPreserveAttributes().size()); try { OptionsParser.parse(new String[] { @@ -545,19 +520,18 @@ public class TestOptionsParser { "hdfs://localhost:9820/target"}); Assert.fail("Invalid preserve attribute"); } - catch (IllegalArgumentException ignore) {} catch (NoSuchElementException ignore) {} - options = OptionsParser.parse(new String[] { - "-f", - "hdfs://localhost:9820/source/first", - "hdfs://localhost:9820/target/"}); - Assert.assertFalse(options.shouldPreserve(FileAttribute.PERMISSION)); - options.preserve(FileAttribute.PERMISSION); - Assert.assertTrue(options.shouldPreserve(FileAttribute.PERMISSION)); + Builder builder = new DistCpOptions.Builder( + new Path("hdfs://localhost:9820/source/first"), + new Path("hdfs://localhost:9820/target/")); + Assert.assertFalse( + builder.build().shouldPreserve(FileAttribute.PERMISSION)); + builder.preserve(FileAttribute.PERMISSION); + Assert.assertTrue(builder.build().shouldPreserve(FileAttribute.PERMISSION)); - options.preserve(FileAttribute.PERMISSION); - Assert.assertTrue(options.shouldPreserve(FileAttribute.PERMISSION)); + builder.preserve(FileAttribute.PERMISSION); + Assert.assertTrue(builder.build().shouldPreserve(FileAttribute.PERMISSION)); } @Test @@ -756,28 +730,25 @@ public class TestOptionsParser { } try { - options = OptionsParser.parse(new String[] { - optionStr, "s1", "s2", "-update", "-delete", + OptionsParser.parse(new String[] { + "-diff", "s1", "s2", "-update", "-delete", "hdfs://localhost:9820/source/first", "hdfs://localhost:9820/target/" }); - assertFalse("-delete should be ignored when " - + optionStr + " is specified", - options.shouldDeleteMissing()); + fail("Should fail as -delete and -diff/-rdiff are mutually exclusive"); } catch (IllegalArgumentException e) { - fail("Got unexpected IllegalArgumentException: " + e.getMessage()); + assertExceptionContains( + "-delete and -diff/-rdiff are mutually exclusive", e); } try { - options = OptionsParser.parse(new String[] { - optionStr, "s1", "s2", "-delete", + OptionsParser.parse(new String[] { + "-diff", "s1", "s2", "-delete", "hdfs://localhost:9820/source/first", "hdfs://localhost:9820/target/" }); - fail(optionStr + " should fail if -update option is not specified"); + fail("Should fail as -delete and -diff/-rdiff are mutually exclusive"); } catch (IllegalArgumentException e) { - assertFalse("-delete should be ignored when -diff is specified", - options.shouldDeleteMissing()); - GenericTestUtils.assertExceptionContains( - "-diff/-rdiff is valid only with -update option", e); + assertExceptionContains( + "-delete and -diff/-rdiff are mutually exclusive", e); } try { @@ -785,10 +756,10 @@ public class TestOptionsParser { "-delete", "-overwrite", "hdfs://localhost:9820/source/first", "hdfs://localhost:9820/target/" }); - fail(optionStr + " should fail if -update option is not specified"); + fail("Should fail as -delete and -diff are mutually exclusive"); } catch (IllegalArgumentException e) { - GenericTestUtils.assertExceptionContains( - "-diff/-rdiff is valid only with -update option", e); + assertExceptionContains( + "-delete and -diff/-rdiff are mutually exclusive", e); } final String optionStrOther = isDiff? "-rdiff" : "-diff"; diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java index 21a14d3abfc..fb1a64db184 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java @@ -19,9 +19,8 @@ package org.apache.hadoop.tools.contract; import static org.apache.hadoop.fs.contract.ContractTestUtils.*; -import static org.junit.Assert.*; -import java.util.Arrays; +import java.util.Collections; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -184,7 +183,8 @@ public abstract class AbstractContractDistCpTest * @throws Exception if there is a failure */ private void runDistCp(Path src, Path dst) throws Exception { - DistCpOptions options = new DistCpOptions(Arrays.asList(src), dst); + DistCpOptions options = new DistCpOptions.Builder( + Collections.singletonList(src), dst).build(); Job job = new DistCp(conf, options).execute(); assertNotNull("Unexpected null job returned from DistCp execution.", job); assertTrue("DistCp job did not complete.", job.isComplete()); diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java index 2452d6fee3d..6ee37ccd6a5 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java @@ -32,6 +32,7 @@ import org.apache.hadoop.mapreduce.task.JobContextImpl; import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat; import org.apache.hadoop.tools.CopyListing; import org.apache.hadoop.tools.DistCpConstants; +import org.apache.hadoop.tools.DistCpContext; import org.apache.hadoop.tools.DistCpOptions; import org.apache.hadoop.tools.DistCpOptions.FileAttribute; import org.apache.hadoop.tools.GlobbedCopyListing; @@ -146,15 +147,16 @@ public class TestCopyCommitter { sourceBase = TestDistCpUtils.createTestSetup(fs, sourcePerm); targetBase = TestDistCpUtils.createTestSetup(fs, initialPerm); - DistCpOptions options = new DistCpOptions(Arrays.asList(new Path(sourceBase)), - new Path("/out")); - options.preserve(FileAttribute.PERMISSION); + final DistCpOptions options = new DistCpOptions.Builder( + Collections.singletonList(new Path(sourceBase)), new Path("/out")) + .preserve(FileAttribute.PERMISSION).build(); options.appendToConf(conf); - options.setTargetPathExists(false); - + final DistCpContext context = new DistCpContext(options); + context.setTargetPathExists(false); + CopyListing listing = new GlobbedCopyListing(conf, CREDENTIALS); Path listingFile = new Path("/tmp1/" + String.valueOf(rand.nextLong())); - listing.buildListing(listingFile, options); + listing.buildListing(listingFile, context); conf.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, targetBase); @@ -197,15 +199,15 @@ public class TestCopyCommitter { String targetBaseAdd = TestDistCpUtils.createTestSetup(fs, FsPermission.getDefault()); fs.rename(new Path(targetBaseAdd), new Path(targetBase)); - DistCpOptions options = new DistCpOptions(Arrays.asList(new Path(sourceBase)), - new Path("/out")); - options.setSyncFolder(true); - options.setDeleteMissing(true); + final DistCpOptions options = new DistCpOptions.Builder( + Collections.singletonList(new Path(sourceBase)), new Path("/out")) + .withSyncFolder(true).withDeleteMissing(true).build(); options.appendToConf(conf); + final DistCpContext context = new DistCpContext(options); CopyListing listing = new GlobbedCopyListing(conf, CREDENTIALS); Path listingFile = new Path("/tmp1/" + String.valueOf(rand.nextLong())); - listing.buildListing(listingFile, options); + listing.buildListing(listingFile, context); conf.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, targetBase); conf.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, targetBase); @@ -266,15 +268,15 @@ public class TestCopyCommitter { TestDistCpUtils.createFile(fs, targetBase + "/9"); TestDistCpUtils.createFile(fs, targetBase + "/A"); - DistCpOptions options = new DistCpOptions(Arrays.asList(new Path(sourceBase)), - new Path("/out")); - options.setSyncFolder(true); - options.setDeleteMissing(true); + final DistCpOptions options = new DistCpOptions.Builder( + Collections.singletonList(new Path(sourceBase)), new Path("/out")) + .withSyncFolder(true).withDeleteMissing(true).build(); options.appendToConf(conf); + final DistCpContext context = new DistCpContext(options); CopyListing listing = new GlobbedCopyListing(conf, CREDENTIALS); Path listingFile = new Path("/tmp1/" + String.valueOf(rand.nextLong())); - listing.buildListing(listingFile, options); + listing.buildListing(listingFile, context); conf.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, targetBase); conf.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, targetBase); diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestUniformSizeInputFormat.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestUniformSizeInputFormat.java index 78e226252d2..5315137fde5 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestUniformSizeInputFormat.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestUniformSizeInputFormat.java @@ -31,6 +31,7 @@ import org.apache.hadoop.mapreduce.task.JobContextImpl; import org.apache.hadoop.mapreduce.lib.input.FileSplit; import org.apache.hadoop.tools.CopyListing; import org.apache.hadoop.tools.CopyListingFileStatus; +import org.apache.hadoop.tools.DistCpContext; import org.apache.hadoop.tools.DistCpOptions; import org.apache.hadoop.tools.StubContext; import org.apache.hadoop.security.Credentials; @@ -74,9 +75,9 @@ public class TestUniformSizeInputFormat { List sourceList = new ArrayList(); sourceList.add(sourcePath); - final DistCpOptions distCpOptions = new DistCpOptions(sourceList, targetPath); - distCpOptions.setMaxMaps(nMaps); - return distCpOptions; + return new DistCpOptions.Builder(sourceList, targetPath) + .maxMaps(nMaps) + .build(); } private static int createFile(String path, int fileSize) throws Exception { @@ -100,14 +101,14 @@ public class TestUniformSizeInputFormat { } public void testGetSplits(int nMaps) throws Exception { - DistCpOptions options = getOptions(nMaps); + DistCpContext context = new DistCpContext(getOptions(nMaps)); Configuration configuration = new Configuration(); configuration.set("mapred.map.tasks", - String.valueOf(options.getMaxMaps())); + String.valueOf(context.getMaxMaps())); Path listFile = new Path(cluster.getFileSystem().getUri().toString() + "/tmp/testGetSplits_1/fileList.seq"); - CopyListing.getCopyListing(configuration, CREDENTIALS, options). - buildListing(listFile, options); + CopyListing.getCopyListing(configuration, CREDENTIALS, context) + .buildListing(listFile, context); JobContext jobContext = new JobContextImpl(configuration, new JobID()); UniformSizeInputFormat uniformSizeInputFormat = new UniformSizeInputFormat(); diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/lib/TestDynamicInputFormat.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/lib/TestDynamicInputFormat.java index bb2dd9d37d1..87290caa8a9 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/lib/TestDynamicInputFormat.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/lib/TestDynamicInputFormat.java @@ -19,6 +19,7 @@ package org.apache.hadoop.tools.mapred.lib; import org.apache.hadoop.tools.DistCpConstants; +import org.apache.hadoop.tools.DistCpContext; import org.junit.Assert; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -84,9 +85,9 @@ public class TestDynamicInputFormat { List sourceList = new ArrayList(); sourceList.add(sourcePath); - DistCpOptions options = new DistCpOptions(sourceList, targetPath); - options.setMaxMaps(NUM_SPLITS); - return options; + return new DistCpOptions.Builder(sourceList, targetPath) + .maxMaps(NUM_SPLITS) + .build(); } private static void createFile(String path) throws Exception { @@ -110,13 +111,13 @@ public class TestDynamicInputFormat { @Test public void testGetSplits() throws Exception { - DistCpOptions options = getOptions(); + final DistCpContext context = new DistCpContext(getOptions()); Configuration configuration = new Configuration(); configuration.set("mapred.map.tasks", - String.valueOf(options.getMaxMaps())); - CopyListing.getCopyListing(configuration, CREDENTIALS, options).buildListing( - new Path(cluster.getFileSystem().getUri().toString() - +"/tmp/testDynInputFormat/fileList.seq"), options); + String.valueOf(context.getMaxMaps())); + CopyListing.getCopyListing(configuration, CREDENTIALS, context) + .buildListing(new Path(cluster.getFileSystem().getUri().toString() + +"/tmp/testDynInputFormat/fileList.seq"), context); JobContext jobContext = new JobContextImpl(configuration, new JobID()); DynamicInputFormat inputFormat = From b8838578ce1f1ae1f93cc6b40cc98b58321ad9ee Mon Sep 17 00:00:00 2001 From: Varun Saxena Date: Sun, 2 Apr 2017 04:37:34 +0530 Subject: [PATCH 180/188] YARN-6414. ATSv2 HBase related tests fail due to guava version upgrade (Haibo Chen via Varun Saxena) --- hadoop-project/pom.xml | 1 + .../pom.xml | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index e55308fed19..f327933b40b 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -51,6 +51,7 @@ 0.8.2.1 1.2.4 2.5.1 + 11.0.2 ${project.version} 1.0.13 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml index d44aa22a38d..afe440fb851 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml @@ -141,6 +141,7 @@ com.google.guava guava + ${hbase-compatible-guava.version} test @@ -369,6 +370,24 @@ + + + org.apache.maven.plugins + maven-enforcer-plugin + + + depcheck + + + true + + + enforce + + + + + From a4b5aa8493e0bd9006f44291d265c28ab86497e1 Mon Sep 17 00:00:00 2001 From: Varun Saxena Date: Sun, 2 Apr 2017 04:54:12 +0530 Subject: [PATCH 181/188] YARN-6377. NMTimelinePublisher#serviceStop does not stop timeline clients (Haibo Chen via Varun Saxena) --- .../timelineservice/NMTimelinePublisher.java | 8 ++++++++ .../timelineservice/TestNMTimelinePublisher.java | 10 +++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java index ce2c6561a2b..8aaae799372 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java @@ -101,6 +101,14 @@ public class NMTimelinePublisher extends CompositeService { this.nodeId = context.getNodeId(); } + @Override + protected void serviceStop() throws Exception { + for(ApplicationId app : appToClientMap.keySet()) { + stopTimelineClient(app); + } + super.serviceStop(); + } + @VisibleForTesting Map getAppToClientMap() { return appToClientMap; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/TestNMTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/TestNMTimelinePublisher.java index e1161229317..0b8eaa97563 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/TestNMTimelinePublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/TestNMTimelinePublisher.java @@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric; import org.apache.hadoop.yarn.client.api.impl.TimelineV2ClientImpl; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; @@ -53,14 +54,21 @@ public class TestNMTimelinePublisher { final DummyTimelineClient timelineClient = new DummyTimelineClient(null); when(context.getNodeId()).thenReturn(NodeId.newInstance("localhost", 0)); when(context.getHttpPort()).thenReturn(0); + + Configuration conf = new Configuration(); + conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); + conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f); + NMTimelinePublisher publisher = new NMTimelinePublisher(context) { public void createTimelineClient(ApplicationId appId) { if (!getAppToClientMap().containsKey(appId)) { + timelineClient.init(getConfig()); + timelineClient.start(); getAppToClientMap().put(appId, timelineClient); } } }; - publisher.init(new Configuration()); + publisher.init(conf); publisher.start(); ApplicationId appId = ApplicationId.newInstance(0, 1); publisher.createTimelineClient(appId); From 845529b3ab338e759665a687eb525fb2cccde7bf Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Mon, 3 Apr 2017 13:06:24 +0900 Subject: [PATCH 182/188] MAPREDUCE-6824. TaskAttemptImpl#createCommonContainerLaunchContext is longer than 150 lines. Contributed by Chris Trezzo. --- .../v2/app/job/impl/TaskAttemptImpl.java | 279 ++++++++++-------- 1 file changed, 150 insertions(+), 129 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java index 43058240707..9ea1b9aa922 100755 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java @@ -755,7 +755,7 @@ public abstract class TaskAttemptImpl implements new HashMap(); // Application environment - Map environment = new HashMap(); + Map environment; // Service data Map serviceData = new HashMap(); @@ -763,157 +763,178 @@ public abstract class TaskAttemptImpl implements // Tokens ByteBuffer taskCredentialsBuffer = ByteBuffer.wrap(new byte[]{}); try { - FileSystem remoteFS = FileSystem.get(conf); - // //////////// Set up JobJar to be localized properly on the remote NM. - String jobJar = conf.get(MRJobConfig.JAR); - if (jobJar != null) { - final Path jobJarPath = new Path(jobJar); - final FileSystem jobJarFs = FileSystem.get(jobJarPath.toUri(), conf); - Path remoteJobJar = jobJarPath.makeQualified(jobJarFs.getUri(), - jobJarFs.getWorkingDirectory()); - LocalResource rc = createLocalResource(jobJarFs, remoteJobJar, - LocalResourceType.PATTERN, LocalResourceVisibility.APPLICATION); - String pattern = conf.getPattern(JobContext.JAR_UNPACK_PATTERN, - JobConf.UNPACK_JAR_PATTERN_DEFAULT).pattern(); - rc.setPattern(pattern); - localResources.put(MRJobConfig.JOB_JAR, rc); - LOG.info("The job-jar file on the remote FS is " - + remoteJobJar.toUri().toASCIIString()); - } else { - // Job jar may be null. For e.g, for pipes, the job jar is the hadoop - // mapreduce jar itself which is already on the classpath. - LOG.info("Job jar is not present. " - + "Not adding any jar to the list of resources."); - } - // //////////// End of JobJar setup + configureJobJar(conf, localResources); - // //////////// Set up JobConf to be localized properly on the remote NM. - Path path = - MRApps.getStagingAreaDir(conf, UserGroupInformation - .getCurrentUser().getShortUserName()); - Path remoteJobSubmitDir = - new Path(path, oldJobId.toString()); - Path remoteJobConfPath = - new Path(remoteJobSubmitDir, MRJobConfig.JOB_CONF_FILE); - localResources.put( - MRJobConfig.JOB_CONF_FILE, - createLocalResource(remoteFS, remoteJobConfPath, - LocalResourceType.FILE, LocalResourceVisibility.APPLICATION)); - LOG.info("The job-conf file on the remote FS is " - + remoteJobConfPath.toUri().toASCIIString()); - // //////////// End of JobConf setup + configureJobConf(conf, localResources, oldJobId); // Setup DistributedCache MRApps.setupDistributedCache(conf, localResources); - // Setup up task credentials buffer - LOG.info("Adding #" + credentials.numberOfTokens() - + " tokens and #" + credentials.numberOfSecretKeys() - + " secret keys for NM use for launching container"); - Credentials taskCredentials = new Credentials(credentials); - - // LocalStorageToken is needed irrespective of whether security is enabled - // or not. - TokenCache.setJobToken(jobToken, taskCredentials); - - DataOutputBuffer containerTokens_dob = new DataOutputBuffer(); - LOG.info("Size of containertokens_dob is " - + taskCredentials.numberOfTokens()); - taskCredentials.writeTokenStorageToStream(containerTokens_dob); taskCredentialsBuffer = - ByteBuffer.wrap(containerTokens_dob.getData(), 0, - containerTokens_dob.getLength()); + configureTokens(jobToken, credentials, serviceData); - // Add shuffle secret key - // The secret key is converted to a JobToken to preserve backwards - // compatibility with an older ShuffleHandler running on an NM. - LOG.info("Putting shuffle token in serviceData"); - byte[] shuffleSecret = TokenCache.getShuffleSecretKey(credentials); - if (shuffleSecret == null) { - LOG.warn("Cannot locate shuffle secret in credentials." - + " Using job token as shuffle secret."); - shuffleSecret = jobToken.getPassword(); - } - Token shuffleToken = new Token( - jobToken.getIdentifier(), shuffleSecret, jobToken.getKind(), - jobToken.getService()); - serviceData.put(ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID, - ShuffleHandler.serializeServiceData(shuffleToken)); + addExternalShuffleProviders(conf, serviceData); - // add external shuffle-providers - if any - Collection shuffleProviders = conf.getStringCollection( - MRJobConfig.MAPREDUCE_JOB_SHUFFLE_PROVIDER_SERVICES); - if (! shuffleProviders.isEmpty()) { - Collection auxNames = conf.getStringCollection( - YarnConfiguration.NM_AUX_SERVICES); + environment = configureEnv(conf); - for (final String shuffleProvider : shuffleProviders) { - if (shuffleProvider.equals(ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID)) { - continue; // skip built-in shuffle-provider that was already inserted with shuffle secret key - } - if (auxNames.contains(shuffleProvider)) { - LOG.info("Adding ShuffleProvider Service: " + shuffleProvider + " to serviceData"); - // This only serves for INIT_APP notifications - // The shuffle service needs to be able to work with the host:port information provided by the AM - // (i.e. shuffle services which require custom location / other configuration are not supported) - serviceData.put(shuffleProvider, ByteBuffer.allocate(0)); - } - else { - throw new YarnRuntimeException("ShuffleProvider Service: " + shuffleProvider + - " was NOT found in the list of aux-services that are available in this NM." + - " You may need to specify this ShuffleProvider as an aux-service in your yarn-site.xml"); - } - } - } - - MRApps.addToEnvironment( - environment, - Environment.CLASSPATH.name(), - getInitialClasspath(conf), conf); - - if (initialAppClasspath != null) { - MRApps.addToEnvironment( - environment, - Environment.APP_CLASSPATH.name(), - initialAppClasspath, conf); - } } catch (IOException e) { throw new YarnRuntimeException(e); } - // Shell - environment.put( - Environment.SHELL.name(), - conf.get( - MRJobConfig.MAPRED_ADMIN_USER_SHELL, - MRJobConfig.DEFAULT_SHELL) - ); - - // Add pwd to LD_LIBRARY_PATH, add this before adding anything else - MRApps.addToEnvironment( - environment, - Environment.LD_LIBRARY_PATH.name(), - MRApps.crossPlatformifyMREnv(conf, Environment.PWD), conf); - - // Add the env variables passed by the admin - MRApps.setEnvFromInputString( - environment, - conf.get( - MRJobConfig.MAPRED_ADMIN_USER_ENV, - MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV), conf - ); - // Construct the actual Container // The null fields are per-container and will be constructed for each // container separately. ContainerLaunchContext container = ContainerLaunchContext.newInstance(localResources, environment, null, - serviceData, taskCredentialsBuffer, applicationACLs); + serviceData, taskCredentialsBuffer, applicationACLs); return container; } + private static Map configureEnv(Configuration conf) + throws IOException { + Map environment = new HashMap(); + MRApps.addToEnvironment(environment, Environment.CLASSPATH.name(), + getInitialClasspath(conf), conf); + + if (initialAppClasspath != null) { + MRApps.addToEnvironment(environment, Environment.APP_CLASSPATH.name(), + initialAppClasspath, conf); + } + + // Shell + environment.put(Environment.SHELL.name(), conf + .get(MRJobConfig.MAPRED_ADMIN_USER_SHELL, MRJobConfig.DEFAULT_SHELL)); + + // Add pwd to LD_LIBRARY_PATH, add this before adding anything else + MRApps.addToEnvironment(environment, Environment.LD_LIBRARY_PATH.name(), + MRApps.crossPlatformifyMREnv(conf, Environment.PWD), conf); + + // Add the env variables passed by the admin + MRApps.setEnvFromInputString(environment, + conf.get(MRJobConfig.MAPRED_ADMIN_USER_ENV, + MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV), + conf); + return environment; + } + + private static void configureJobJar(Configuration conf, + Map localResources) throws IOException { + // Set up JobJar to be localized properly on the remote NM. + String jobJar = conf.get(MRJobConfig.JAR); + if (jobJar != null) { + final Path jobJarPath = new Path(jobJar); + final FileSystem jobJarFs = FileSystem.get(jobJarPath.toUri(), conf); + Path remoteJobJar = jobJarPath.makeQualified(jobJarFs.getUri(), + jobJarFs.getWorkingDirectory()); + LocalResource rc = createLocalResource(jobJarFs, remoteJobJar, + LocalResourceType.PATTERN, LocalResourceVisibility.APPLICATION); + String pattern = conf.getPattern(JobContext.JAR_UNPACK_PATTERN, + JobConf.UNPACK_JAR_PATTERN_DEFAULT).pattern(); + rc.setPattern(pattern); + localResources.put(MRJobConfig.JOB_JAR, rc); + LOG.info("The job-jar file on the remote FS is " + + remoteJobJar.toUri().toASCIIString()); + } else { + // Job jar may be null. For e.g, for pipes, the job jar is the hadoop + // mapreduce jar itself which is already on the classpath. + LOG.info("Job jar is not present. " + + "Not adding any jar to the list of resources."); + } + } + + private static void configureJobConf(Configuration conf, + Map localResources, + final org.apache.hadoop.mapred.JobID oldJobId) throws IOException { + // Set up JobConf to be localized properly on the remote NM. + Path path = MRApps.getStagingAreaDir(conf, + UserGroupInformation.getCurrentUser().getShortUserName()); + Path remoteJobSubmitDir = new Path(path, oldJobId.toString()); + Path remoteJobConfPath = + new Path(remoteJobSubmitDir, MRJobConfig.JOB_CONF_FILE); + FileSystem remoteFS = FileSystem.get(conf); + localResources.put(MRJobConfig.JOB_CONF_FILE, + createLocalResource(remoteFS, remoteJobConfPath, LocalResourceType.FILE, + LocalResourceVisibility.APPLICATION)); + LOG.info("The job-conf file on the remote FS is " + + remoteJobConfPath.toUri().toASCIIString()); + } + + private static ByteBuffer configureTokens(Token jobToken, + Credentials credentials, + Map serviceData) throws IOException { + // Setup up task credentials buffer + LOG.info("Adding #" + credentials.numberOfTokens() + " tokens and #" + + credentials.numberOfSecretKeys() + + " secret keys for NM use for launching container"); + Credentials taskCredentials = new Credentials(credentials); + + // LocalStorageToken is needed irrespective of whether security is enabled + // or not. + TokenCache.setJobToken(jobToken, taskCredentials); + + DataOutputBuffer containerTokens_dob = new DataOutputBuffer(); + LOG.info( + "Size of containertokens_dob is " + taskCredentials.numberOfTokens()); + taskCredentials.writeTokenStorageToStream(containerTokens_dob); + ByteBuffer taskCredentialsBuffer = + ByteBuffer.wrap(containerTokens_dob.getData(), 0, + containerTokens_dob.getLength()); + + // Add shuffle secret key + // The secret key is converted to a JobToken to preserve backwards + // compatibility with an older ShuffleHandler running on an NM. + LOG.info("Putting shuffle token in serviceData"); + byte[] shuffleSecret = TokenCache.getShuffleSecretKey(credentials); + if (shuffleSecret == null) { + LOG.warn("Cannot locate shuffle secret in credentials." + + " Using job token as shuffle secret."); + shuffleSecret = jobToken.getPassword(); + } + Token shuffleToken = + new Token(jobToken.getIdentifier(), shuffleSecret, + jobToken.getKind(), jobToken.getService()); + serviceData.put(ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID, + ShuffleHandler.serializeServiceData(shuffleToken)); + return taskCredentialsBuffer; + } + + private static void addExternalShuffleProviders(Configuration conf, + Map serviceData) { + // add external shuffle-providers - if any + Collection shuffleProviders = conf.getStringCollection( + MRJobConfig.MAPREDUCE_JOB_SHUFFLE_PROVIDER_SERVICES); + if (!shuffleProviders.isEmpty()) { + Collection auxNames = + conf.getStringCollection(YarnConfiguration.NM_AUX_SERVICES); + + for (final String shuffleProvider : shuffleProviders) { + if (shuffleProvider + .equals(ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID)) { + continue; // skip built-in shuffle-provider that was already inserted + // with shuffle secret key + } + if (auxNames.contains(shuffleProvider)) { + LOG.info("Adding ShuffleProvider Service: " + shuffleProvider + + " to serviceData"); + // This only serves for INIT_APP notifications + // The shuffle service needs to be able to work with the host:port + // information provided by the AM + // (i.e. shuffle services which require custom location / other + // configuration are not supported) + serviceData.put(shuffleProvider, ByteBuffer.allocate(0)); + } else { + throw new YarnRuntimeException("ShuffleProvider Service: " + + shuffleProvider + + " was NOT found in the list of aux-services that are " + + "available in this NM. You may need to specify this " + + "ShuffleProvider as an aux-service in your yarn-site.xml"); + } + } + } + } + static ContainerLaunchContext createContainerLaunchContext( Map applicationACLs, Configuration conf, Token jobToken, Task remoteTask, From bc7aff7cec07bbc3fed63a00c8f1584c34670998 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Mon, 3 Apr 2017 07:32:27 -0700 Subject: [PATCH 183/188] HDFS-11515. -du throws ConcurrentModificationException. Contributed by Istvan Fajth, Wei-Chiu Chuang. --- .../DirectoryWithSnapshotFeature.java | 5 ++ .../snapshot/TestRenameWithSnapshots.java | 6 +- .../snapshot/TestSnapshotDeletion.java | 75 +++++++++++++++++++ 3 files changed, 84 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java index 9addbfa7ace..984067990ca 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java @@ -633,6 +633,11 @@ public class DirectoryWithSnapshotFeature implements INode.Feature { for(DirectoryDiff d : diffs) { for(INode deletedNode : d.getChildrenDiff().getList(ListType.DELETED)) { context.reportDeletedSnapshottedNode(deletedNode); + if (deletedNode.isDirectory()){ + DirectoryWithSnapshotFeature sf = + deletedNode.asDirectory().getDirectoryWithSnapshotFeature(); + sf.computeContentSummary4Snapshot(context); + } } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java index d1b3aa6f870..d06c384fa2f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java @@ -26,6 +26,7 @@ import static org.mockito.Matchers.anyBoolean; import static org.mockito.Matchers.anyObject; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.spy; +import static org.apache.hadoop.test.GenericTestUtils.getTestDir; import java.io.File; import java.io.IOException; @@ -2429,7 +2430,7 @@ public class TestRenameWithSnapshots { */ @Test (timeout=300000) public void testDu() throws Exception { - File tempFile = File.createTempFile("testDu-", ".tmp"); + File tempFile = File.createTempFile("testDu-", ".tmp", getTestDir()); tempFile.deleteOnExit(); final FileSystem localfs = FileSystem.getLocal(conf); @@ -2539,7 +2540,8 @@ public class TestRenameWithSnapshots { */ @Test (timeout=300000) public void testDuMultipleDirs() throws Exception { - File tempFile = File.createTempFile("testDuMultipleDirs-", "" + ".tmp"); + File tempFile = File.createTempFile("testDuMultipleDirs-", ".tmp", + getTestDir()); tempFile.deleteOnExit(); final FileSystem localfs = FileSystem.getLocal(conf); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java index ca53788e98d..7926e44d0cf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java @@ -27,6 +27,7 @@ import java.io.PrintStream; import java.security.PrivilegedAction; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FsShell; import org.apache.hadoop.fs.Path; @@ -1232,4 +1233,78 @@ public class TestSnapshotDeletion { // make sure bar has been cleaned from inodeMap Assert.assertNull(fsdir.getInode(fileId)); } + + /** + * Test for HDFS-11515. + * In a scenario where a directory with subdirectories is removed from the + * file system after taking a snapshot on one of its ancestors, du command + * fails with a ConcurrentModificationException until a new snapshot is taken, + * or the old snapshots are removed. + * This test is testing this scenario with checks on the space consumed + * calculation. + * + * @throws Exception + */ + @Test(timeout = 180000) + public void testDuWithRmdirInSnapshots() throws Exception { + final Path parent = new Path("/testDuWithRmdirInSnapshots"); + final Path snapshotDir = new Path(parent, "snapshotDir"); + final Path dir1 = new Path(snapshotDir, "d1"); //snapshotDir/d1 + final Path dir2 = new Path(snapshotDir, "d2"); //snapshotDir/d2 + final Path dir4 = new Path(dir2, "d4"); //snapshotDir/d2/d4 + final Path dir3 = new Path(snapshotDir, "d3"); //snapshotDir/d3 + final Path dir5 = new Path(dir3, "d5"); //snapshotDir/d3/d5 + final Path aFileOutsideSnapshots = new Path(parent, "aFile"); + final Path aFileInsideSnapshots = new Path(dir5, "aFile"); + + final String snapshotName = "s1"; + final String snapshotName2 = "s2"; + + final long spaceConsumed = BLOCKSIZE * REPLICATION; + final long spaceConsumed2 = 2 * spaceConsumed; + ContentSummary summary = null; + + DFSTestUtil.createFile(hdfs, aFileOutsideSnapshots, + BLOCKSIZE, REPLICATION, 0); + summary = hdfs.getContentSummary(parent); + assertEquals("Du is wrong even with one file without further ado.", + spaceConsumed, summary.getSpaceConsumed()); + + hdfs.mkdirs(snapshotDir); + hdfs.allowSnapshot(snapshotDir); + hdfs.mkdirs(dir1); + + hdfs.createSnapshot(snapshotDir, snapshotName); + + hdfs.mkdirs(dir4); + hdfs.mkdirs(dir5); + DFSTestUtil.createFile(hdfs, aFileInsideSnapshots, + BLOCKSIZE, REPLICATION, 0); + summary = hdfs.getContentSummary(parent); + assertEquals("Du is wrong with 2 files added to the file system.", + spaceConsumed2, summary.getSpaceConsumed()); + + hdfs.createSnapshot(snapshotDir, snapshotName2); + + hdfs.delete(dir2, true); + hdfs.delete(dir3, true); + + summary = hdfs.getContentSummary(parent); + assertEquals("Snapshot file count is not matching expected value.", + 1, summary.getSnapshotFileCount()); + assertEquals("Snapshot directory count is not matching expected value.", + 4, summary.getSnapshotDirectoryCount()); + assertEquals("Consumed space does not matching expected value.", + spaceConsumed, summary.getSnapshotSpaceConsumed()); + assertEquals("Snapshot length is not matching expected value.", + BLOCKSIZE, summary.getSnapshotLength()); + assertEquals("File count is not matching expected value.", + 2, summary.getFileCount()); + assertEquals("Directory count is not matching expected value.", + 7, summary.getDirectoryCount()); + assertEquals("Consumed space is not matching expected value.", + spaceConsumed2, summary.getSpaceConsumed()); + assertEquals("Length is not matching expected value.", + 2 * BLOCKSIZE, summary.getLength()); + } } From bbd68478d5743b3b2911bf3febed7daa89479e45 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Mon, 3 Apr 2017 07:57:28 -0700 Subject: [PATCH 184/188] HDFS-11598. Improve -setrep for Erasure Coded files. Contributed by Yiqun Lin. --- .../hadoop/fs/shell/SetReplication.java | 17 +++++-- .../hadoop/hdfs/TestSetrepIncreasing.java | 44 +++++++++++++++++++ 2 files changed, 57 insertions(+), 4 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java index 2231c58f2b5..16e6e929ede 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java @@ -85,11 +85,20 @@ class SetReplication extends FsCommand { } if (item.stat.isFile()) { - if (!item.fs.setReplication(item.path, newRep)) { - throw new IOException("Could not set replication for: " + item); + // Do the checking if the file is erasure coded since + // replication factor for an EC file is meaningless. + if (!item.stat.isErasureCoded()) { + if (!item.fs.setReplication(item.path, newRep)) { + throw new IOException("Could not set replication for: " + item); + } + out.println("Replication " + newRep + " set: " + item); + if (waitOpt) { + waitList.add(item); + } + } else { + out.println("Did not set replication for: " + item + + ", because it's an erasure coded file."); } - out.println("Replication " + newRep + " set: " + item); - if (waitOpt) waitList.add(item); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java index fee30b5e58b..50d7b2756f9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSetrepIncreasing.java @@ -20,7 +20,9 @@ package org.apache.hadoop.hdfs; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.PrintStream; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; @@ -28,6 +30,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FsShell; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset; import org.junit.Test; @@ -102,4 +105,45 @@ public class TestSetrepIncreasing { cluster.shutdown(); } } + + @Test + public void testSetRepOnECFile() throws Exception { + ClientProtocol client; + Configuration conf = new HdfsConfiguration(); + conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, + StripedFileTestUtil.getDefaultECPolicy().getName()); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1) + .build(); + cluster.waitActive(); + client = NameNodeProxies.createProxy(conf, + cluster.getFileSystem(0).getUri(), ClientProtocol.class).getProxy(); + client.setErasureCodingPolicy("/", + StripedFileTestUtil.getDefaultECPolicy().getName()); + + FileSystem dfs = cluster.getFileSystem(); + try { + Path d = new Path("/tmp"); + dfs.mkdirs(d); + Path f = new Path(d, "foo"); + dfs.createNewFile(f); + FileStatus file = dfs.getFileStatus(f); + assertTrue(file.isErasureCoded()); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + System.setOut(new PrintStream(out)); + String[] args = {"-setrep", "2", "" + f}; + FsShell shell = new FsShell(); + shell.setConf(conf); + assertEquals(0, shell.run(args)); + assertTrue( + out.toString().contains("Did not set replication for: /tmp/foo")); + + // verify the replication factor of the EC file + file = dfs.getFileStatus(f); + assertEquals(1, file.getReplication()); + } finally { + dfs.close(); + cluster.shutdown(); + } + } } From 5faa949b782be48ef400d2eb1695f420455de764 Mon Sep 17 00:00:00 2001 From: Mingliang Liu Date: Mon, 3 Apr 2017 11:07:14 -0700 Subject: [PATCH 185/188] HADOOP-14268. Fix markdown itemization in hadoop-aws documents. Contributed by Akira Ajisaka --- .../hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md | 2 ++ .../hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md | 1 + 2 files changed, 3 insertions(+) diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index 82c35885301..18c0cebdcba 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -41,6 +41,7 @@ The specifics of using these filesystems are documented in this section. See also: + * [Testing](testing.html) * [Troubleshooting S3a](troubleshooting_s3a.html) @@ -99,6 +100,7 @@ access to the data. Anyone with the credentials can not only read your datasets —they can delete them. Do not inadvertently share these credentials through means such as + 1. Checking in to SCM any configuration files containing the secrets. 1. Logging them to a console, as they invariably end up being seen. 1. Defining filesystem URIs with the credentials in the URL, such as diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md index 79551a3290d..39ca8f42b0b 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md @@ -643,6 +643,7 @@ located. New tests are always welcome. Bear in mind that we need to keep costs and test time down, which is done by + * Not duplicating tests. * Being efficient in your use of Hadoop API calls. * Isolating large/slow tests into the "scale" test group. From 6eba79232f36b36e0196163adc8fe4219a6b6bf9 Mon Sep 17 00:00:00 2001 From: Chris Douglas Date: Mon, 3 Apr 2017 20:13:14 -0700 Subject: [PATCH 186/188] HADOOP-14271. Correct spelling of 'occurred' and variants. Contributed by Yeliang Cang --- .../org/apache/hadoop/util/Progressable.java | 2 +- .../hadoop/util/UTF8ByteArrayUtils.java | 4 +-- .../src/main/native/gtest/gtest-all.cc | 2 +- .../main/native/gtest/include/gtest/gtest.h | 2 +- .../hadoop/test/MultithreadedTestUtil.java | 2 +- .../apache/hadoop/hdfs/DFSInputStream.java | 2 +- .../hadoop/hdfs/DFSStripedInputStream.java | 2 +- .../hadoop/fs/http/server/FSOperations.java | 34 +++++++++---------- .../fsdataset/impl/BlockPoolSlice.java | 2 +- .../datanode/fsdataset/impl/FsVolumeImpl.java | 2 +- .../mapreduce/lib/jobcontrol/JobControl.java | 2 +- .../fs/azure/BlockBlobAppendStream.java | 2 +- .../hadoop/streaming/StreamKeyValUtil.java | 8 ++--- .../TestDistributedShell.java | 30 ++++++++-------- .../launcher/TestContainerLaunch.java | 12 +++---- .../scheduler/capacity/TestParentQueue.java | 18 +++++----- 16 files changed, 63 insertions(+), 63 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progressable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progressable.java index 495ca82b76e..201ee5c41c9 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progressable.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progressable.java @@ -28,7 +28,7 @@ import org.apache.hadoop.classification.InterfaceStability; * to explicitly report progress to the Hadoop framework. This is especially * important for operations which take significant amount of time since, * in-lieu of the reported progress, the framework has to assume that an error - * has occured and time-out the operation.

    + * has occurred and time-out the operation.

    */ @InterfaceAudience.Public @InterfaceStability.Stable diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/UTF8ByteArrayUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/UTF8ByteArrayUtils.java index 2a804c618e1..069494f0f38 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/UTF8ByteArrayUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/UTF8ByteArrayUtils.java @@ -30,7 +30,7 @@ public class UTF8ByteArrayUtils { * @param start starting offset * @param end ending position * @param b the byte to find - * @return position that first byte occures otherwise -1 + * @return position that first byte occurs, otherwise -1 */ public static int findByte(byte [] utf, int start, int end, byte b) { for(int i=start; itrue if the delete operation was successful, * false otherwise. * - * @throws IOException thrown if an IO error occured. + * @throws IOException thrown if an IO error occurred. */ @Override public JSONObject execute(FileSystem fs) throws IOException { @@ -583,7 +583,7 @@ public class FSOperations { * * @return a Map object (JSON friendly) with the file checksum. * - * @throws IOException thrown if an IO error occured. + * @throws IOException thrown if an IO error occurred. */ @Override public Map execute(FileSystem fs) throws IOException { @@ -640,7 +640,7 @@ public class FSOperations { * * @return a JSON object with the user home directory. * - * @throws IOException thrown if an IO error occured. + * @throws IOException thrown if an IO error occurred. */ @Override @SuppressWarnings("unchecked") @@ -765,7 +765,7 @@ public class FSOperations { * @return true if the mkdirs operation was successful, * false otherwise. * - * @throws IOException thrown if an IO error occured. + * @throws IOException thrown if an IO error occurred. */ @Override public JSONObject execute(FileSystem fs) throws IOException { @@ -799,7 +799,7 @@ public class FSOperations { * * @return The inputstream of the file. * - * @throws IOException thrown if an IO error occured. + * @throws IOException thrown if an IO error occurred. */ @Override public InputStream execute(FileSystem fs) throws IOException { @@ -837,7 +837,7 @@ public class FSOperations { * @return true if the rename operation was successful, * false otherwise. * - * @throws IOException thrown if an IO error occured. + * @throws IOException thrown if an IO error occurred. */ @Override public JSONObject execute(FileSystem fs) throws IOException { @@ -876,7 +876,7 @@ public class FSOperations { * * @return void. * - * @throws IOException thrown if an IO error occured. + * @throws IOException thrown if an IO error occurred. */ @Override public Void execute(FileSystem fs) throws IOException { @@ -913,7 +913,7 @@ public class FSOperations { * * @return void. * - * @throws IOException thrown if an IO error occured. + * @throws IOException thrown if an IO error occurred. */ @Override public Void execute(FileSystem fs) throws IOException { @@ -1186,7 +1186,7 @@ public class FSOperations { * @return true if the replication value was set, * false otherwise. * - * @throws IOException thrown if an IO error occured. + * @throws IOException thrown if an IO error occurred. */ @Override @SuppressWarnings("unchecked") @@ -1228,7 +1228,7 @@ public class FSOperations { * * @return void. * - * @throws IOException thrown if an IO error occured. + * @throws IOException thrown if an IO error occurred. */ @Override public Void execute(FileSystem fs) throws IOException { @@ -1314,7 +1314,7 @@ public class FSOperations { * * @return Map a map object (JSON friendly) with the xattr names. * - * @throws IOException thrown if an IO error occured. + * @throws IOException thrown if an IO error occurred. */ @Override public Map execute(FileSystem fs) throws IOException { @@ -1353,7 +1353,7 @@ public class FSOperations { * * @return Map a map object (JSON friendly) with the xattrs. * - * @throws IOException thrown if an IO error occured. + * @throws IOException thrown if an IO error occurred. */ @Override public Map execute(FileSystem fs) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java index c8df300ff6b..c17ef366271 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java @@ -822,7 +822,7 @@ class BlockPoolSlice { } catch (Exception e) { // Any exception we need to revert back to read from disk // Log the error and return false - LOG.info("Exception occured while reading the replicas cache file: " + LOG.info("Exception occurred while reading the replicas cache file: " + replicaFile.getPath(), e ); return false; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java index f6e6a59aa43..b948fb788a4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java @@ -1323,7 +1323,7 @@ public class FsVolumeImpl implements FsVolumeSpi { fileNames = fileIoProvider.listDirectory( this, dir, BlockDirFilter.INSTANCE); } catch (IOException ioe) { - LOG.warn("Exception occured while compiling report: ", ioe); + LOG.warn("Exception occurred while compiling report: ", ioe); // Volume error check moved to FileIoProvider. // Ignore this directory and proceed. return report; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/jobcontrol/JobControl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/jobcontrol/JobControl.java index b0b7a3c119f..e5399b5f744 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/jobcontrol/JobControl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/jobcontrol/JobControl.java @@ -276,7 +276,7 @@ public class JobControl implements Runnable { } synchronized private void failAllJobs(Throwable t) { - String message = "Unexpected System Error Occured: "+ + String message = "Unexpected System Error Occurred: "+ StringUtils.stringifyException(t); Iterator it = jobsInProgress.iterator(); while(it.hasNext()) { diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java index a7e286c917a..afb9379c3ca 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/BlockBlobAppendStream.java @@ -346,7 +346,7 @@ public class BlockBlobAppendStream extends OutputStream { try { if (!ioThreadPool.awaitTermination(10, TimeUnit.MINUTES)) { - LOG.error("Time out occured while waiting for IO request to finish in append" + LOG.error("Time out occurred while waiting for IO request to finish in append" + " for blob : {}", key); NativeAzureFileSystemHelper.logAllLiveStackTraces(); throw new IOException("Timed out waiting for IO requests to finish"); diff --git a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamKeyValUtil.java b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamKeyValUtil.java index 75e05dc0708..fba45b11f24 100644 --- a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamKeyValUtil.java +++ b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamKeyValUtil.java @@ -26,11 +26,11 @@ import org.apache.hadoop.util.LineReader; public class StreamKeyValUtil { /** - * Find the first occured tab in a UTF-8 encoded string + * Find the first occurred tab in a UTF-8 encoded string * @param utf a byte array containing a UTF-8 encoded string * @param start starting offset * @param length no. of bytes - * @return position that first tab occures otherwise -1 + * @return position that first tab occurres otherwise -1 */ public static int findTab(byte [] utf, int start, int length) { for(int i=start; i<(start+length); i++) { @@ -41,9 +41,9 @@ public class StreamKeyValUtil { return -1; } /** - * Find the first occured tab in a UTF-8 encoded string + * Find the first occurred tab in a UTF-8 encoded string * @param utf a byte array containing a UTF-8 encoded string - * @return position that first tab occures otherwise -1 + * @return position that first tab occurres otherwise -1 */ public static int findTab(byte [] utf) { return org.apache.hadoop.util.UTF8ByteArrayUtils.findNthByte(utf, 0, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java index 300ea67497c..ef21c8786c5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java @@ -544,17 +544,17 @@ public class TestDistributedShell { Assert.assertEquals( "Container created event needs to be published atleast once", 1, - getNumOfStringOccurences(containerEntityFile, + getNumOfStringOccurrences(containerEntityFile, ContainerMetricsConstants.CREATED_EVENT_TYPE)); // to avoid race condition of testcase, atleast check 4 times with sleep // of 500ms - long numOfContainerFinishedOccurences = 0; + long numOfContainerFinishedOccurrences = 0; for (int i = 0; i < 4; i++) { - numOfContainerFinishedOccurences = - getNumOfStringOccurences(containerEntityFile, + numOfContainerFinishedOccurrences = + getNumOfStringOccurrences(containerEntityFile, ContainerMetricsConstants.FINISHED_EVENT_TYPE); - if (numOfContainerFinishedOccurences > 0) { + if (numOfContainerFinishedOccurrences > 0) { break; } else { Thread.sleep(500L); @@ -563,7 +563,7 @@ public class TestDistributedShell { Assert.assertEquals( "Container finished event needs to be published atleast once", 1, - numOfContainerFinishedOccurences); + numOfContainerFinishedOccurrences); // Verify RM posting Application life cycle Events are getting published String appMetricsTimestampFileName = @@ -576,17 +576,17 @@ public class TestDistributedShell { Assert.assertEquals( "Application created event should be published atleast once", 1, - getNumOfStringOccurences(appEntityFile, + getNumOfStringOccurrences(appEntityFile, ApplicationMetricsConstants.CREATED_EVENT_TYPE)); // to avoid race condition of testcase, atleast check 4 times with sleep // of 500ms - long numOfStringOccurences = 0; + long numOfStringOccurrences = 0; for (int i = 0; i < 4; i++) { - numOfStringOccurences = - getNumOfStringOccurences(appEntityFile, + numOfStringOccurrences = + getNumOfStringOccurrences(appEntityFile, ApplicationMetricsConstants.FINISHED_EVENT_TYPE); - if (numOfStringOccurences > 0) { + if (numOfStringOccurrences > 0) { break; } else { Thread.sleep(500L); @@ -595,7 +595,7 @@ public class TestDistributedShell { Assert.assertEquals( "Application finished event should be published atleast once", 1, - numOfStringOccurences); + numOfStringOccurrences); // Verify RM posting AppAttempt life cycle Events are getting published String appAttemptMetricsTimestampFileName = @@ -609,13 +609,13 @@ public class TestDistributedShell { Assert.assertEquals( "AppAttempt register event should be published atleast once", 1, - getNumOfStringOccurences(appAttemptEntityFile, + getNumOfStringOccurrences(appAttemptEntityFile, AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE)); Assert.assertEquals( "AppAttempt finished event should be published atleast once", 1, - getNumOfStringOccurences(appAttemptEntityFile, + getNumOfStringOccurrences(appAttemptEntityFile, AppAttemptMetricsConstants.FINISHED_EVENT_TYPE)); } finally { FileUtils.deleteDirectory(tmpRootFolder.getParentFile()); @@ -636,7 +636,7 @@ public class TestDistributedShell { return entityFile; } - private long getNumOfStringOccurences(File entityFile, String searchString) + private long getNumOfStringOccurrences(File entityFile, String searchString) throws IOException { BufferedReader reader = null; String strLine; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index 23b99d9a7e5..8dcf4be41f8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -641,8 +641,8 @@ public class TestContainerLaunch extends BaseContainerManagerTest { ContainerLaunch launch = new ContainerLaunch(context, conf, dispatcher, exec, app, container, dirsHandler, containerManager); launch.call(); - Assert.assertTrue("ContainerExitEvent should have occured", - eventHandler.isContainerExitEventOccured()); + Assert.assertTrue("ContainerExitEvent should have occurred", + eventHandler.isContainerExitEventOccurred()); } private static class ContainerExitHandler implements EventHandler { @@ -652,15 +652,15 @@ public class TestContainerLaunch extends BaseContainerManagerTest { this.testForMultiFile = testForMultiFile; } - boolean containerExitEventOccured = false; + boolean containerExitEventOccurred = false; - public boolean isContainerExitEventOccured() { - return containerExitEventOccured; + public boolean isContainerExitEventOccurred() { + return containerExitEventOccurred; } public void handle(Event event) { if (event instanceof ContainerExitEvent) { - containerExitEventOccured = true; + containerExitEventOccurred = true; ContainerExitEvent exitEvent = (ContainerExitEvent) event; Assert.assertEquals(ContainerEventType.CONTAINER_EXITED_WITH_FAILURE, exitEvent.getType()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java index c4b7a0d4031..cdbbc519857 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java @@ -343,43 +343,43 @@ public class TestParentQueue { csConf.setCapacity(Q_B, 70.5F); Map queues = new HashMap(); - boolean exceptionOccured = false; + boolean exceptionOccurred = false; try { CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); } catch (IllegalArgumentException ie) { - exceptionOccured = true; + exceptionOccurred = true; } - if (!exceptionOccured) { + if (!exceptionOccurred) { Assert.fail("Capacity is more then 100% so should be failed."); } csConf.setCapacity(Q_A, 30); csConf.setCapacity(Q_B, 70); - exceptionOccured = false; + exceptionOccurred = false; queues.clear(); try { CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); } catch (IllegalArgumentException ie) { - exceptionOccured = true; + exceptionOccurred = true; } - if (exceptionOccured) { + if (exceptionOccurred) { Assert.fail("Capacity is 100% so should not be failed."); } csConf.setCapacity(Q_A, 30); csConf.setCapacity(Q_B, 70.005F); - exceptionOccured = false; + exceptionOccurred = false; queues.clear(); try { CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); } catch (IllegalArgumentException ie) { - exceptionOccured = true; + exceptionOccurred = true; } - if (exceptionOccured) { + if (exceptionOccurred) { Assert .fail("Capacity is under PRECISION which is .05% so should not be failed."); } From 27a44b60c1a55cabbf1873bca62db4e321c901ca Mon Sep 17 00:00:00 2001 From: Mingliang Liu Date: Tue, 4 Apr 2017 11:03:59 -0700 Subject: [PATCH 187/188] HADOOP-14272. Azure: WasbRemoteCallHelper should use String equals for comparison. Contributed by Santhosh G Nayak --- .../java/org/apache/hadoop/fs/azure/WasbRemoteCallHelper.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbRemoteCallHelper.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbRemoteCallHelper.java index 09ea0847ee4..b43e5aec2a0 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbRemoteCallHelper.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/WasbRemoteCallHelper.java @@ -88,7 +88,8 @@ class WasbRemoteCallHelper { } Header contentTypeHeader = response.getFirstHeader("Content-Type"); - if (contentTypeHeader == null || contentTypeHeader.getValue() != APPLICATION_JSON) { + if (contentTypeHeader == null + || !APPLICATION_JSON.equals(contentTypeHeader.getValue())) { throw new WasbRemoteCallException(getRequest.getURI().toString() + ":" + "Content-Type mismatch: expected: " + APPLICATION_JSON + ", got " + ((contentTypeHeader!=null) ? contentTypeHeader.getValue() : "NULL") From 56ab02eed9b61e1c80605104dfc4c87fc6abac96 Mon Sep 17 00:00:00 2001 From: Mingliang Liu Date: Tue, 4 Apr 2017 11:10:28 -0700 Subject: [PATCH 188/188] HADOOP-14273. Azure: NativeAzureFileSystem should respect config for kerberosSupportEnabled flag. Contributed by Santhosh G Nayak --- .../java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java index 1e6f00eb337..5469944f9b3 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java @@ -1287,6 +1287,8 @@ public class NativeAzureFileSystem extends FileSystem { this.azureAuthorization = useSecureMode && conf.getBoolean(KEY_AZURE_AUTHORIZATION, DEFAULT_AZURE_AUTHORIZATION); + this.kerberosSupportEnabled = + conf.getBoolean(Constants.AZURE_KERBEROS_SUPPORT_PROPERTY_NAME, false); if (this.azureAuthorization) {
    Node Health Report {{model.node.healthReport}}
    Node Manager Start Time{{model.node.nmStartupTime}}
    Node Manager Start Time{{model.node.nmStartupTime}}
    Node Manager Version {{model.node.nodeManagerBuildVersion}}