From 0da148dae74f8b969a4a694d080d001f6b653f7f Mon Sep 17 00:00:00 2001 From: Kihwal Lee Date: Thu, 19 Jun 2014 19:05:25 +0000 Subject: [PATCH 001/112] HDFS-3848. A Bug in recoverLeaseInternal method of FSNameSystem class. Contributed by Hooman Peiro Sajjad and Chen He. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1604011 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hdfs/server/namenode/FSNamesystem.java | 4 +-- .../hadoop/hdfs/TestLeaseRecovery2.java | 25 ++++++++++++++++++- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index a88324336f3..86f193074b1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -670,6 +670,9 @@ Release 2.5.0 - UNRELEASED HDFS-6563. NameNode cannot save fsimage in certain circumstances when snapshots are in use. (atm) + HDFS-3848. A Bug in recoverLeaseInternal method of FSNameSystem class + (Hooman Peiro Sajjad and Chen He via kihwal) + BREAKDOWN OF HDFS-2006 SUBTASKS AND RELATED JIRAS HDFS-6299. Protobuf for XAttr and client-side implementation. (Yi Liu via umamahesh) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 3846add44ee..ad7b4d61285 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -2584,10 +2584,10 @@ private void recoverLeaseInternal(INodeFile fileInode, // We found the lease for this file. And surprisingly the original // holder is trying to recreate this file. This should never occur. // + if (!force && lease != null) { Lease leaseFile = leaseManager.getLeaseByPath(src); - if ((leaseFile != null && leaseFile.equals(lease)) || - lease.getHolder().equals(holder)) { + if (leaseFile != null && leaseFile.equals(lease)) { throw new AlreadyBeingCreatedException( "failed to create file " + src + " for " + holder + " for client " + clientMachine + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java index 04a0d2298e7..6d981fbfd18 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java @@ -153,6 +153,15 @@ public void testImmediateRecoveryOfLease() throws Exception { verifyFile(dfs, filepath1, actual, size); } + @Test + public void testLeaseRecoverByAnotherUser() throws Exception { + byte [] actual = new byte[FILE_SIZE]; + cluster.setLeasePeriod(SHORT_LEASE_PERIOD, LONG_LEASE_PERIOD); + Path filepath = createFile("/immediateRecoverLease-x", 0, true); + recoverLeaseUsingCreate2(filepath); + verifyFile(dfs, filepath, actual, 0); + } + private Path createFile(final String filestr, final int size, final boolean triggerLeaseRenewerInterrupt) throws IOException, InterruptedException { @@ -196,7 +205,7 @@ private FileSystem getFSAsAnotherUser(final Configuration c) } private void recoverLeaseUsingCreate(Path filepath) - throws IOException, InterruptedException { + throws IOException, InterruptedException { FileSystem dfs2 = getFSAsAnotherUser(conf); for(int i = 0; i < 10; i++) { AppendTestUtil.LOG.info("i=" + i); @@ -216,6 +225,20 @@ private void recoverLeaseUsingCreate(Path filepath) fail("recoverLeaseUsingCreate failed"); } + private void recoverLeaseUsingCreate2(Path filepath) + throws Exception { + FileSystem dfs2 = getFSAsAnotherUser(conf); + int size = AppendTestUtil.nextInt(FILE_SIZE); + DistributedFileSystem dfsx = (DistributedFileSystem) dfs2; + //create file using dfsx + Path filepath2 = new Path("/immediateRecoverLease-x2"); + FSDataOutputStream stm = dfsx.create(filepath2, true, BUF_SIZE, + REPLICATION_NUM, BLOCK_SIZE); + assertTrue(dfsx.dfs.exists("/immediateRecoverLease-x2")); + try {Thread.sleep(10000);} catch (InterruptedException e) {} + dfsx.append(filepath); + } + private void verifyFile(FileSystem dfs, Path filepath, byte[] actual, int size) throws IOException { AppendTestUtil.LOG.info("Lease for file " + filepath + " is recovered. " From 9ff3836a367737d6dfcb12f50c8bd2f1b2233e37 Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Thu, 19 Jun 2014 19:39:29 +0000 Subject: [PATCH 002/112] HDFS-6549. Add support for accessing the NFS gateway from the AIX NFS client. Contributed by Aaron T. Myers. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1604022 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop/hdfs/nfs/conf/NfsConfigKeys.java | 4 +- .../hadoop/hdfs/nfs/nfs3/OpenFileCtx.java | 36 +++++++++++--- .../hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java | 49 ++++++++++++++++--- .../hadoop/hdfs/nfs/nfs3/WriteManager.java | 7 ++- .../hadoop/hdfs/nfs/nfs3/TestWrites.java | 25 +++++++++- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../src/site/apt/HdfsNfsGateway.apt.vm | 19 +++++++ 7 files changed, 124 insertions(+), 19 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/conf/NfsConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/conf/NfsConfigKeys.java index d1543b8a084..2f65ce4cba4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/conf/NfsConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/conf/NfsConfigKeys.java @@ -55,4 +55,6 @@ public class NfsConfigKeys { public static final String DFS_NFS_PORT_MONITORING_DISABLED_KEY = "nfs.port.monitoring.disabled"; public static final boolean DFS_NFS_PORT_MONITORING_DISABLED_DEFAULT = true; -} \ No newline at end of file + public static final String AIX_COMPAT_MODE_KEY = "nfs.aix.compatibility.mode.enabled"; + public static final boolean AIX_COMPAT_MODE_DEFAULT = false; +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java index e2ab31787b8..cf44af56758 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java @@ -95,6 +95,7 @@ static enum COMMIT_STATUS { */ private AtomicLong nextOffset; private final HdfsDataOutputStream fos; + private final boolean aixCompatMode; // It's updated after each sync to HDFS private Nfs3FileAttributes latestAttr; @@ -199,8 +200,15 @@ private long updateNonSequentialWriteInMemory(long count) { OpenFileCtx(HdfsDataOutputStream fos, Nfs3FileAttributes latestAttr, String dumpFilePath, DFSClient client, IdUserGroup iug) { + this(fos, latestAttr, dumpFilePath, client, iug, false); + } + + OpenFileCtx(HdfsDataOutputStream fos, Nfs3FileAttributes latestAttr, + String dumpFilePath, DFSClient client, IdUserGroup iug, + boolean aixCompatMode) { this.fos = fos; this.latestAttr = latestAttr; + this.aixCompatMode = aixCompatMode; // We use the ReverseComparatorOnMin as the comparator of the map. In this // way, we first dump the data with larger offset. In the meanwhile, we // retrieve the last element to write back to HDFS. @@ -780,15 +788,29 @@ synchronized COMMIT_STATUS checkCommitInternal(long commitOffset, } if (commitOffset > 0) { - if (commitOffset > flushed) { - if (!fromRead) { - CommitCtx commitCtx = new CommitCtx(commitOffset, channel, xid, - preOpAttr); - pendingCommits.put(commitOffset, commitCtx); + if (aixCompatMode) { + // The AIX NFS client misinterprets RFC-1813 and will always send 4096 + // for the commitOffset even if fewer bytes than that have ever (or will + // ever) be sent by the client. So, if in AIX compatibility mode, we + // will always DO_SYNC if the number of bytes to commit have already all + // been flushed, else we will fall through to the logic below which + // checks for pending writes in the case that we're being asked to + // commit more bytes than have so far been flushed. See HDFS-6549 for + // more info. + if (commitOffset <= flushed) { + return COMMIT_STATUS.COMMIT_DO_SYNC; } - return COMMIT_STATUS.COMMIT_WAIT; } else { - return COMMIT_STATUS.COMMIT_DO_SYNC; + if (commitOffset > flushed) { + if (!fromRead) { + CommitCtx commitCtx = new CommitCtx(commitOffset, channel, xid, + preOpAttr); + pendingCommits.put(commitOffset, commitCtx); + } + return COMMIT_STATUS.COMMIT_WAIT; + } else { + return COMMIT_STATUS.COMMIT_DO_SYNC; + } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java index 4fc14ba1a0d..446e722a213 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java @@ -153,6 +153,7 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface { private final short replication; private final long blockSize; private final int bufferSize; + private final boolean aixCompatMode; private Statistics statistics; private String writeDumpDir; // The dir save dump files @@ -170,8 +171,11 @@ public RpcProgramNfs3(NfsConfiguration config, DatagramSocket registrationSocket config.set(FsPermission.UMASK_LABEL, "000"); iug = new IdUserGroup(config); + aixCompatMode = config.getBoolean( + NfsConfigKeys.AIX_COMPAT_MODE_KEY, + NfsConfigKeys.AIX_COMPAT_MODE_DEFAULT); exports = NfsExports.getInstance(config); - writeManager = new WriteManager(iug, config); + writeManager = new WriteManager(iug, config, aixCompatMode); clientCache = new DFSClientCache(config); replication = (short) config.getInt(DFSConfigKeys.DFS_REPLICATION_KEY, DFSConfigKeys.DFS_REPLICATION_DEFAULT); @@ -900,7 +904,8 @@ preOpDirAttr, new WccData(Nfs3Utils.getWccAttr(preOpDirAttr), // Add open stream OpenFileCtx openFileCtx = new OpenFileCtx(fos, postOpObjAttr, - writeDumpDir + "/" + postOpObjAttr.getFileId(), dfsClient, iug); + writeDumpDir + "/" + postOpObjAttr.getFileId(), dfsClient, iug, + aixCompatMode); fileHandle = new FileHandle(postOpObjAttr.getFileId()); if (!writeManager.addOpenFileStream(fileHandle, openFileCtx)) { LOG.warn("Can't add more stream, close it." @@ -1438,9 +1443,24 @@ public READDIR3Response readdir(XDR xdr, SecurityHandler securityHandler, } long cookieVerf = request.getCookieVerf(); if ((cookieVerf != 0) && (cookieVerf != dirStatus.getModificationTime())) { - LOG.error("CookierVerf mismatch. request cookierVerf:" + cookieVerf - + " dir cookieVerf:" + dirStatus.getModificationTime()); - return new READDIR3Response(Nfs3Status.NFS3ERR_BAD_COOKIE); + if (aixCompatMode) { + // The AIX NFS client misinterprets RFC-1813 and will repeatedly send + // the same cookieverf value even across VFS-level readdir calls, + // instead of getting a new cookieverf for every VFS-level readdir + // call, and reusing the cookieverf only in the event that multiple + // incremental NFS-level readdir calls must be made to fetch all of + // the directory entries. This means that whenever a readdir call is + // made by an AIX NFS client for a given directory, and that directory + // is subsequently modified, thus changing its mtime, no later readdir + // calls will succeed from AIX for that directory until the FS is + // unmounted/remounted. See HDFS-6549 for more info. + LOG.warn("AIX compatibility mode enabled, ignoring cookieverf " + + "mismatches."); + } else { + LOG.error("CookieVerf mismatch. request cookieVerf: " + cookieVerf + + " dir cookieVerf: " + dirStatus.getModificationTime()); + return new READDIR3Response(Nfs3Status.NFS3ERR_BAD_COOKIE); + } } if (cookie == 0) { @@ -1588,9 +1608,22 @@ READDIRPLUS3Response readdirplus(XDR xdr, SecurityHandler securityHandler, } long cookieVerf = request.getCookieVerf(); if ((cookieVerf != 0) && (cookieVerf != dirStatus.getModificationTime())) { - LOG.error("CookierVerf mismatch. request cookierVerf:" + cookieVerf - + " dir cookieVerf:" + dirStatus.getModificationTime()); - return new READDIRPLUS3Response(Nfs3Status.NFS3ERR_BAD_COOKIE); + if (aixCompatMode) { + // The AIX NFS client misinterprets RFC-1813 and will repeatedly send + // the same cookieverf value even across VFS-level readdir calls, + // instead of getting a new cookieverf for every VFS-level readdir + // call. This means that whenever a readdir call is made by an AIX NFS + // client for a given directory, and that directory is subsequently + // modified, thus changing its mtime, no later readdir calls will + // succeed for that directory from AIX until the FS is + // unmounted/remounted. See HDFS-6549 for more info. + LOG.warn("AIX compatibility mode enabled, ignoring cookieverf " + + "mismatches."); + } else { + LOG.error("cookieverf mismatch. request cookieverf: " + cookieVerf + + " dir cookieverf: " + dirStatus.getModificationTime()); + return new READDIRPLUS3Response(Nfs3Status.NFS3ERR_BAD_COOKIE); + } } if (cookie == 0) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java index 2cd8b22c9dc..5f2ded744d5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java @@ -58,6 +58,7 @@ public class WriteManager { private boolean asyncDataServiceStarted = false; private final int maxStreams; + private final boolean aixCompatMode; /** * The time limit to wait for accumulate reordered sequential writes to the @@ -79,9 +80,11 @@ boolean addOpenFileStream(FileHandle h, OpenFileCtx ctx) { return fileContextCache.put(h, ctx); } - WriteManager(IdUserGroup iug, final NfsConfiguration config) { + WriteManager(IdUserGroup iug, final NfsConfiguration config, + boolean aixCompatMode) { this.iug = iug; this.config = config; + this.aixCompatMode = aixCompatMode; streamTimeout = config.getLong(NfsConfigKeys.DFS_NFS_STREAM_TIMEOUT_KEY, NfsConfigKeys.DFS_NFS_STREAM_TIMEOUT_DEFAULT); LOG.info("Stream timeout is " + streamTimeout + "ms."); @@ -175,7 +178,7 @@ void handleWrite(DFSClient dfsClient, WRITE3Request request, Channel channel, String writeDumpDir = config.get(NfsConfigKeys.DFS_NFS_FILE_DUMP_DIR_KEY, NfsConfigKeys.DFS_NFS_FILE_DUMP_DIR_DEFAULT); openFileCtx = new OpenFileCtx(fos, latestAttr, writeDumpDir + "/" - + fileHandle.getFileId(), dfsClient, iug); + + fileHandle.getFileId(), dfsClient, iug, aixCompatMode); if (!addOpenFileStream(fileHandle, openFileCtx)) { LOG.info("Can't add new stream. Close it. Tell client to retry."); diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java index 90cfc54febb..3945b298f5e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java @@ -190,6 +190,29 @@ public void testCheckCommit() throws IOException { ret = ctx.checkCommit(dfsClient, 0, ch, 1, attr, false); Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_FINISHED); } + + @Test + public void testCheckCommitAixCompatMode() throws IOException { + DFSClient dfsClient = Mockito.mock(DFSClient.class); + Nfs3FileAttributes attr = new Nfs3FileAttributes(); + HdfsDataOutputStream fos = Mockito.mock(HdfsDataOutputStream.class); + + // Last argument "true" here to enable AIX compatibility mode. + OpenFileCtx ctx = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, + new IdUserGroup(new NfsConfiguration()), true); + + // Test fall-through to pendingWrites check in the event that commitOffset + // is greater than the number of bytes we've so far flushed. + Mockito.when(fos.getPos()).thenReturn((long) 2); + COMMIT_STATUS status = ctx.checkCommitInternal(5, null, 1, attr, false); + Assert.assertTrue(status == COMMIT_STATUS.COMMIT_FINISHED); + + // Test the case when we actually have received more bytes than we're trying + // to commit. + Mockito.when(fos.getPos()).thenReturn((long) 10); + status = ctx.checkCommitInternal(5, null, 1, attr, false); + Assert.assertTrue(status == COMMIT_STATUS.COMMIT_DO_SYNC); + } @Test // Validate all the commit check return codes OpenFileCtx.COMMIT_STATUS, which @@ -207,7 +230,7 @@ public void testCheckCommitFromRead() throws IOException { FileHandle h = new FileHandle(1); // fake handle for "/dumpFilePath" COMMIT_STATUS ret; - WriteManager wm = new WriteManager(new IdUserGroup(config), config); + WriteManager wm = new WriteManager(new IdUserGroup(config), config, false); assertTrue(wm.addOpenFileStream(h, ctx)); // Test inactive open file context diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 86f193074b1..0a717773c07 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -673,6 +673,9 @@ Release 2.5.0 - UNRELEASED HDFS-3848. A Bug in recoverLeaseInternal method of FSNameSystem class (Hooman Peiro Sajjad and Chen He via kihwal) + HDFS-6549. Add support for accessing the NFS gateway from the AIX NFS + client. (atm) + BREAKDOWN OF HDFS-2006 SUBTASKS AND RELATED JIRAS HDFS-6299. Protobuf for XAttr and client-side implementation. (Yi Liu via umamahesh) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm index 09cdb5819dd..54544cff46f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm @@ -88,6 +88,25 @@ HDFS NFS Gateway ---- + The AIX NFS client has a {{{https://issues.apache.org/jira/browse/HDFS-6549}few known issues}} + that prevent it from working correctly by default with the HDFS NFS + Gateway. If you want to be able to access the HDFS NFS Gateway from AIX, you + should set the following configuration setting to enable work-arounds for these + issues: + +---- + + nfs.aix.compatibility.mode.enabled + true + +---- + + Note that regular, non-AIX clients should NOT enable AIX compatibility mode. + The work-arounds implemented by AIX compatibility mode effectively disable + safeguards to ensure that listing of directory contents via NFS returns + consistent results, and that all data sent to the NFS server can be assured to + have been committed. + It's strongly recommended for the users to update a few configuration properties based on their use cases. All the related configuration properties can be added or updated in hdfs-site.xml. From 38e2322d84d54896eac23afec0a1434629b8c8b2 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Thu, 19 Jun 2014 22:18:03 +0000 Subject: [PATCH 003/112] HADOOP-10696. Add optional attributes to KeyProvider Options and Metadata. (tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1604041 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 ++ .../crypto/key/JavaKeyStoreProvider.java | 2 +- .../apache/hadoop/crypto/key/KeyProvider.java | 48 ++++++++++++++++++- .../hadoop/crypto/key/UserProvider.java | 2 +- .../crypto/key/kms/KMSClientProvider.java | 9 +++- .../crypto/key/kms/KMSRESTConstants.java | 1 + .../hadoop/crypto/key/TestKeyProvider.java | 17 +++++-- .../hadoop/crypto/key/kms/server/KMS.java | 5 +- .../key/kms/server/KMSServerJSONUtils.java | 1 + .../hadoop/crypto/key/kms/server/TestKMS.java | 43 +++++++++++++++++ .../kms/server/TestKMSCacheKeyProvider.java | 2 +- 11 files changed, 122 insertions(+), 11 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index f732e80bc2f..fe0ef94e0af 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -152,6 +152,9 @@ Trunk (Unreleased) HADOOP-10607. Create API to separate credential/password storage from applications. (Larry McCay via omalley) + HADOOP-10696. Add optional attributes to KeyProvider Options and Metadata. + (tucu) + BUG FIXES HADOOP-9451. Fault single-layer config if node group topology is enabled. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java index ecf90ad82c9..0f22f6343ae 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java @@ -270,7 +270,7 @@ public KeyVersion createKey(String name, byte[] material, e); } Metadata meta = new Metadata(options.getCipher(), options.getBitLength(), - options.getDescription(), new Date(), 1); + options.getDescription(), options.getAttributes(), new Date(), 1); if (options.getBitLength() != 8 * material.length) { throw new IOException("Wrong key length. Required " + options.getBitLength() + ", but got " + (8 * material.length)); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java index 85115996335..01d7b697ae1 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java @@ -26,8 +26,11 @@ import java.net.URI; import java.security.NoSuchAlgorithmException; import java.text.MessageFormat; +import java.util.Collections; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; import com.google.gson.stream.JsonReader; import com.google.gson.stream.JsonWriter; @@ -107,18 +110,22 @@ public static class Metadata { private final static String CREATED_FIELD = "created"; private final static String DESCRIPTION_FIELD = "description"; private final static String VERSIONS_FIELD = "versions"; + private final static String ATTRIBUTES_FIELD = "attributes"; private final String cipher; private final int bitLength; private final String description; private final Date created; private int versions; + private Map attributes; - protected Metadata(String cipher, int bitLength, - String description, Date created, int versions) { + protected Metadata(String cipher, int bitLength, String description, + Map attributes, Date created, int versions) { this.cipher = cipher; this.bitLength = bitLength; this.description = description; + this.attributes = (attributes == null || attributes.isEmpty()) + ? null : attributes; this.created = created; this.versions = versions; } @@ -141,6 +148,11 @@ public String getCipher() { return cipher; } + @SuppressWarnings("unchecked") + public Map getAttributes() { + return (attributes == null) ? Collections.EMPTY_MAP : attributes; + } + /** * Get the algorithm from the cipher. * @return the algorithm name @@ -188,6 +200,13 @@ protected byte[] serialize() throws IOException { if (description != null) { writer.name(DESCRIPTION_FIELD).value(description); } + if (attributes != null && attributes.size() > 0) { + writer.name(ATTRIBUTES_FIELD).beginObject(); + for (Map.Entry attribute : attributes.entrySet()) { + writer.name(attribute.getKey()).value(attribute.getValue()); + } + writer.endObject(); + } writer.name(VERSIONS_FIELD).value(versions); writer.endObject(); writer.flush(); @@ -208,6 +227,7 @@ protected Metadata(byte[] bytes) throws IOException { Date created = null; int versions = 0; String description = null; + Map attributes = null; JsonReader reader = new JsonReader(new InputStreamReader (new ByteArrayInputStream(bytes))); try { @@ -224,6 +244,13 @@ protected Metadata(byte[] bytes) throws IOException { versions = reader.nextInt(); } else if (DESCRIPTION_FIELD.equals(field)) { description = reader.nextString(); + } else if (ATTRIBUTES_FIELD.equalsIgnoreCase(field)) { + reader.beginObject(); + attributes = new HashMap(); + while (reader.hasNext()) { + attributes.put(reader.nextName(), reader.nextString()); + } + reader.endObject(); } } reader.endObject(); @@ -234,6 +261,7 @@ protected Metadata(byte[] bytes) throws IOException { this.bitLength = bitLength; this.created = created; this.description = description; + this.attributes = attributes; this.versions = versions; } } @@ -245,6 +273,7 @@ public static class Options { private String cipher; private int bitLength; private String description; + private Map attributes; public Options(Configuration conf) { cipher = conf.get(DEFAULT_CIPHER_NAME, DEFAULT_CIPHER); @@ -266,6 +295,16 @@ public Options setDescription(String description) { return this; } + public Options setAttributes(Map attributes) { + if (attributes != null) { + if (attributes.containsKey(null)) { + throw new IllegalArgumentException("attributes cannot have a NULL key"); + } + this.attributes = new HashMap(attributes); + } + return this; + } + public String getCipher() { return cipher; } @@ -277,6 +316,11 @@ public int getBitLength() { public String getDescription() { return description; } + + @SuppressWarnings("unchecked") + public Map getAttributes() { + return (attributes == null) ? Collections.EMPTY_MAP : attributes; + } } /** diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/UserProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/UserProvider.java index 371938b372c..6cfb46bd719 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/UserProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/UserProvider.java @@ -89,7 +89,7 @@ public synchronized KeyVersion createKey(String name, byte[] material, options.getBitLength() + ", but got " + (8 * material.length)); } Metadata meta = new Metadata(options.getCipher(), options.getBitLength(), - options.getDescription(), new Date(), 1); + options.getDescription(), options.getAttributes(), new Date(), 1); cache.put(name, meta); String versionName = buildVersionName(name, 0); credentials.addSecretKey(nameT, meta.serialize()); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java index 024a192be90..41c1f60be7f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java @@ -83,6 +83,7 @@ private static KeyVersion parseJSONKeyVersion(Map valueMap) { return keyVersion; } + @SuppressWarnings("unchecked") private static Metadata parseJSONMetadata(Map valueMap) { Metadata metadata = null; if (!valueMap.isEmpty()) { @@ -90,6 +91,7 @@ private static Metadata parseJSONMetadata(Map valueMap) { (String) valueMap.get(KMSRESTConstants.CIPHER_FIELD), (Integer) valueMap.get(KMSRESTConstants.LENGTH_FIELD), (String) valueMap.get(KMSRESTConstants.DESCRIPTION_FIELD), + (Map) valueMap.get(KMSRESTConstants.ATTRIBUTES_FIELD), new Date((Long) valueMap.get(KMSRESTConstants.CREATED_FIELD)), (Integer) valueMap.get(KMSRESTConstants.VERSIONS_FIELD)); } @@ -351,8 +353,8 @@ public List getKeys() throws IOException { public static class KMSMetadata extends Metadata { public KMSMetadata(String cipher, int bitLength, String description, - Date created, int versions) { - super(cipher, bitLength, description, created, versions); + Map attributes, Date created, int versions) { + super(cipher, bitLength, description, attributes, created, versions); } } @@ -416,6 +418,9 @@ private KeyVersion createKeyInternal(String name, byte[] material, jsonKey.put(KMSRESTConstants.DESCRIPTION_FIELD, options.getDescription()); } + if (options.getAttributes() != null && !options.getAttributes().isEmpty()) { + jsonKey.put(KMSRESTConstants.ATTRIBUTES_FIELD, options.getAttributes()); + } URL url = createURL(KMSRESTConstants.KEYS_RESOURCE, null, null, null); HttpURLConnection conn = createConnection(url, HTTP_POST); conn.setRequestProperty(CONTENT_TYPE, APPLICATION_JSON_MIME); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSRESTConstants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSRESTConstants.java index 3d2ea349974..807cba7fbba 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSRESTConstants.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSRESTConstants.java @@ -42,6 +42,7 @@ public class KMSRESTConstants { public static final String CIPHER_FIELD = "cipher"; public static final String LENGTH_FIELD = "length"; public static final String DESCRIPTION_FIELD = "description"; + public static final String ATTRIBUTES_FIELD = "attributes"; public static final String CREATED_FIELD = "created"; public static final String VERSIONS_FIELD = "versions"; public static final String MATERIAL_FIELD = "material"; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java index 352fb3a57c1..7da16757638 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java @@ -30,7 +30,9 @@ import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; @@ -73,7 +75,7 @@ public void testMetadata() throws Exception { DateFormat format = new SimpleDateFormat("y/m/d"); Date date = format.parse("2013/12/25"); KeyProvider.Metadata meta = new KeyProvider.Metadata("myCipher", 100, null, - date, 123); + null, date, 123); assertEquals("myCipher", meta.getCipher()); assertEquals(100, meta.getBitLength()); assertNull(meta.getDescription()); @@ -83,6 +85,7 @@ public void testMetadata() throws Exception { assertEquals(meta.getCipher(), second.getCipher()); assertEquals(meta.getBitLength(), second.getBitLength()); assertNull(second.getDescription()); + assertTrue(second.getAttributes().isEmpty()); assertEquals(meta.getCreated(), second.getCreated()); assertEquals(meta.getVersions(), second.getVersions()); int newVersion = second.addVersion(); @@ -93,17 +96,21 @@ public void testMetadata() throws Exception { //Metadata with description format = new SimpleDateFormat("y/m/d"); date = format.parse("2013/12/25"); + Map attributes = new HashMap(); + attributes.put("a", "A"); meta = new KeyProvider.Metadata("myCipher", 100, - "description", date, 123); + "description", attributes, date, 123); assertEquals("myCipher", meta.getCipher()); assertEquals(100, meta.getBitLength()); assertEquals("description", meta.getDescription()); + assertEquals(attributes, meta.getAttributes()); assertEquals(date, meta.getCreated()); assertEquals(123, meta.getVersions()); second = new KeyProvider.Metadata(meta.serialize()); assertEquals(meta.getCipher(), second.getCipher()); assertEquals(meta.getBitLength(), second.getBitLength()); assertEquals(meta.getDescription(), second.getDescription()); + assertEquals(meta.getAttributes(), second.getAttributes()); assertEquals(meta.getCreated(), second.getCreated()); assertEquals(meta.getVersions(), second.getVersions()); newVersion = second.addVersion(); @@ -117,15 +124,19 @@ public void testOptions() throws Exception { Configuration conf = new Configuration(); conf.set(KeyProvider.DEFAULT_CIPHER_NAME, "myCipher"); conf.setInt(KeyProvider.DEFAULT_BITLENGTH_NAME, 512); + Map attributes = new HashMap(); + attributes.put("a", "A"); KeyProvider.Options options = KeyProvider.options(conf); assertEquals("myCipher", options.getCipher()); assertEquals(512, options.getBitLength()); options.setCipher("yourCipher"); options.setDescription("description"); + options.setAttributes(attributes); options.setBitLength(128); assertEquals("yourCipher", options.getCipher()); assertEquals(128, options.getBitLength()); assertEquals("description", options.getDescription()); + assertEquals(attributes, options.getAttributes()); options = KeyProvider.options(new Configuration()); assertEquals(KeyProvider.DEFAULT_CIPHER, options.getCipher()); assertEquals(KeyProvider.DEFAULT_BITLENGTH, options.getBitLength()); @@ -167,7 +178,7 @@ public List getKeyVersions(String name) @Override public Metadata getMetadata(String name) throws IOException { - return new Metadata(CIPHER, 128, "description", new Date(), 0); + return new Metadata(CIPHER, 128, "description", null, new Date(), 0); } @Override diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java index de8d8443d39..3446c787b88 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java +++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java @@ -103,6 +103,7 @@ private static URI getKeyURI(String name) throws URISyntaxException { @Path(KMSRESTConstants.KEYS_RESOURCE) @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) + @SuppressWarnings("unchecked") public Response createKey(@Context SecurityContext securityContext, Map jsonKey) throws Exception { KMSWebApp.getAdminCallsMeter().mark(); @@ -116,7 +117,8 @@ public Response createKey(@Context SecurityContext securityContext, ? (Integer) jsonKey.get(KMSRESTConstants.LENGTH_FIELD) : 0; String description = (String) jsonKey.get(KMSRESTConstants.DESCRIPTION_FIELD); - + Map attributes = (Map) + jsonKey.get(KMSRESTConstants.ATTRIBUTES_FIELD); if (material != null) { assertAccess(KMSACLs.Type.SET_KEY_MATERIAL, user, CREATE_KEY + " with user provided material", name); @@ -130,6 +132,7 @@ public Response createKey(@Context SecurityContext securityContext, options.setBitLength(length); } options.setDescription(description); + options.setAttributes(attributes); KeyProvider.KeyVersion keyVersion = (material != null) ? provider.createKey(name, Base64.decodeBase64(material), options) diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java index cc995cdf12b..9131a189adb 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java +++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java @@ -61,6 +61,7 @@ public static Map toJSON(String keyName, KeyProvider.Metadata meta) { json.put(KMSRESTConstants.CIPHER_FIELD, meta.getCipher()); json.put(KMSRESTConstants.LENGTH_FIELD, meta.getBitLength()); json.put(KMSRESTConstants.DESCRIPTION_FIELD, meta.getDescription()); + json.put(KMSRESTConstants.ATTRIBUTES_FIELD, meta.getAttributes()); json.put(KMSRESTConstants.CREATED_FIELD, meta.getCreated().getTime()); json.put(KMSRESTConstants.VERSIONS_FIELD, diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java index 70aa59896c6..0959dce47e7 100644 --- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java +++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java @@ -490,6 +490,49 @@ public Void call() throws Exception { // getKeysMetadata() empty Assert.assertEquals(0, kp.getKeysMetadata().length); + // createKey() no description, no tags + options = new KeyProvider.Options(conf); + options.setCipher("AES/CTR/NoPadding"); + options.setBitLength(128); + kp.createKey("k2", options); + KeyProvider.Metadata meta = kp.getMetadata("k2"); + Assert.assertNull(meta.getDescription()); + Assert.assertTrue(meta.getAttributes().isEmpty()); + + // createKey() description, no tags + options = new KeyProvider.Options(conf); + options.setCipher("AES/CTR/NoPadding"); + options.setBitLength(128); + options.setDescription("d"); + kp.createKey("k3", options); + meta = kp.getMetadata("k3"); + Assert.assertEquals("d", meta.getDescription()); + Assert.assertTrue(meta.getAttributes().isEmpty()); + + Map attributes = new HashMap(); + attributes.put("a", "A"); + + // createKey() no description, tags + options = new KeyProvider.Options(conf); + options.setCipher("AES/CTR/NoPadding"); + options.setBitLength(128); + options.setAttributes(attributes); + kp.createKey("k4", options); + meta = kp.getMetadata("k4"); + Assert.assertNull(meta.getDescription()); + Assert.assertEquals(attributes, meta.getAttributes()); + + // createKey() description, tags + options = new KeyProvider.Options(conf); + options.setCipher("AES/CTR/NoPadding"); + options.setBitLength(128); + options.setDescription("d"); + options.setAttributes(attributes); + kp.createKey("k5", options); + meta = kp.getMetadata("k5"); + Assert.assertEquals("d", meta.getDescription()); + Assert.assertEquals(attributes, meta.getAttributes()); + return null; } }); diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSCacheKeyProvider.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSCacheKeyProvider.java index 110b0c95027..72b219191bc 100644 --- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSCacheKeyProvider.java +++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSCacheKeyProvider.java @@ -102,7 +102,7 @@ public void testDeleteKey() throws Exception { Mockito.when(mockProv.getCurrentKey(Mockito.eq("k1"))).thenReturn(mockKey); Mockito.when(mockProv.getKeyVersion(Mockito.eq("k1@0"))).thenReturn(mockKey); Mockito.when(mockProv.getMetadata(Mockito.eq("k1"))).thenReturn( - new KMSClientProvider.KMSMetadata("c", 0, "l", new Date(), 1)); + new KMSClientProvider.KMSMetadata("c", 0, "l", null, new Date(), 1)); KeyProvider cache = new KMSCacheKeyProvider(mockProv, 100); Assert.assertEquals(mockKey, cache.getCurrentKey("k1")); Mockito.verify(mockProv, Mockito.times(1)).getCurrentKey(Mockito.eq("k1")); From 46dc32e12568c5e254a3a2f2664095dc9de8bd55 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Thu, 19 Jun 2014 23:06:40 +0000 Subject: [PATCH 004/112] HDFS-6312. WebHdfs HA failover is broken on secure clusters. (daryn via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1604045 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java | 2 -- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 0a717773c07..baf65464ef5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -745,6 +745,9 @@ Release 2.5.0 - UNRELEASED HDFS-6492. Support create-time xattrs and atomically setting multiple xattrs. (wang) + HDFS-6312. WebHdfs HA failover is broken on secure clusters. + (daryn via tucu) + Release 2.4.1 - 2014-06-23 INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index a3238df74a9..f8dcc398bd2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -344,8 +344,6 @@ private synchronized InetSocketAddress getCurrentNNAddr() { */ private synchronized void resetStateToFailOver() { currentNNAddrIndex = (currentNNAddrIndex + 1) % nnAddrs.length; - delegationToken = null; - tokenAspect.reset(); } /** From af6c91a80c299f87af8c42fa685448b596b7615a Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Fri, 20 Jun 2014 02:38:00 +0000 Subject: [PATCH 005/112] HDFS-6403. Add metrics for log warnings reported by JVM pauses. Contributed by Yongjun Zhang. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1604074 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop/metrics2/source/JvmMetrics.java | 15 +++++++++++ .../metrics2/source/JvmMetricsInfo.java | 5 +++- .../apache/hadoop/util/JvmPauseMonitor.java | 27 ++++++++++++++++--- .../hadoop-common/src/site/apt/Metrics.apt.vm | 8 ++++++ .../metrics2/source/TestJvmMetrics.java | 9 ++++++- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hadoop/hdfs/server/datanode/DataNode.java | 3 ++- .../datanode/metrics/DataNodeMetrics.java | 16 +++++++---- .../hadoop/hdfs/server/namenode/NameNode.java | 3 ++- .../namenode/metrics/NameNodeMetrics.java | 15 ++++++++--- 10 files changed, 89 insertions(+), 15 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java index 750499139a6..c62caf39510 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java @@ -38,6 +38,7 @@ import org.apache.hadoop.metrics2.lib.Interns; import static org.apache.hadoop.metrics2.source.JvmMetricsInfo.*; import static org.apache.hadoop.metrics2.impl.MsInfo.*; +import org.apache.hadoop.util.JvmPauseMonitor; /** * JVM and logging related metrics. @@ -65,6 +66,7 @@ synchronized JvmMetrics init(String processName, String sessionId) { ManagementFactory.getGarbageCollectorMXBeans(); final ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean(); final String processName, sessionId; + private JvmPauseMonitor pauseMonitor = null; final ConcurrentHashMap gcInfoCache = new ConcurrentHashMap(); @@ -73,6 +75,10 @@ synchronized JvmMetrics init(String processName, String sessionId) { this.sessionId = sessionId; } + public void setPauseMonitor(final JvmPauseMonitor pauseMonitor) { + this.pauseMonitor = pauseMonitor; + } + public static JvmMetrics create(String processName, String sessionId, MetricsSystem ms) { return ms.register(JvmMetrics.name(), JvmMetrics.description(), @@ -120,6 +126,15 @@ private void getGcUsage(MetricsRecordBuilder rb) { } rb.addCounter(GcCount, count) .addCounter(GcTimeMillis, timeMillis); + + if (pauseMonitor != null) { + rb.addCounter(GcNumWarnThresholdExceeded, + pauseMonitor.getNumGcWarnThreadholdExceeded()); + rb.addCounter(GcNumInfoThresholdExceeded, + pauseMonitor.getNumGcInfoThresholdExceeded()); + rb.addCounter(GcTotalExtraSleepTime, + pauseMonitor.getTotalGcExtraSleepTime()); + } } private MetricsInfo[] getGcInfo(String gcName) { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetricsInfo.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetricsInfo.java index 64a59b6ee56..55bb41720ed 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetricsInfo.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetricsInfo.java @@ -48,7 +48,10 @@ public enum JvmMetricsInfo implements MetricsInfo { LogFatal("Total number of fatal log events"), LogError("Total number of error log events"), LogWarn("Total number of warning log events"), - LogInfo("Total number of info log events"); + LogInfo("Total number of info log events"), + GcNumWarnThresholdExceeded("Number of times that the GC warn threshold is exceeded"), + GcNumInfoThresholdExceeded("Number of times that the GC info threshold is exceeded"), + GcTotalExtraSleepTime("Total GC extra sleep time in milliseconds"); private final String desc; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java index f7932a6c8bb..e8af45e7462 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java @@ -62,10 +62,13 @@ public class JvmPauseMonitor { "jvm.pause.info-threshold.ms"; private static final long INFO_THRESHOLD_DEFAULT = 1000; - + private long numGcWarnThresholdExceeded = 0; + private long numGcInfoThresholdExceeded = 0; + private long totalGcExtraSleepTime = 0; + private Thread monitorThread; private volatile boolean shouldRun = true; - + public JvmPauseMonitor(Configuration conf) { this.warnThresholdMs = conf.getLong(WARN_THRESHOLD_KEY, WARN_THRESHOLD_DEFAULT); this.infoThresholdMs = conf.getLong(INFO_THRESHOLD_KEY, INFO_THRESHOLD_DEFAULT); @@ -87,6 +90,22 @@ public void stop() { Thread.currentThread().interrupt(); } } + + public boolean isStarted() { + return monitorThread != null; + } + + public long getNumGcWarnThreadholdExceeded() { + return numGcWarnThresholdExceeded; + } + + public long getNumGcInfoThresholdExceeded() { + return numGcInfoThresholdExceeded; + } + + public long getTotalGcExtraSleepTime() { + return totalGcExtraSleepTime; + } private String formatMessage(long extraSleepTime, Map gcTimesAfterSleep, @@ -166,13 +185,15 @@ public void run() { Map gcTimesAfterSleep = getGcTimes(); if (extraSleepTime > warnThresholdMs) { + ++numGcWarnThresholdExceeded; LOG.warn(formatMessage( extraSleepTime, gcTimesAfterSleep, gcTimesBeforeSleep)); } else if (extraSleepTime > infoThresholdMs) { + ++numGcInfoThresholdExceeded; LOG.info(formatMessage( extraSleepTime, gcTimesAfterSleep, gcTimesBeforeSleep)); } - + totalGcExtraSleepTime += extraSleepTime; gcTimesBeforeSleep = gcTimesAfterSleep; } } diff --git a/hadoop-common-project/hadoop-common/src/site/apt/Metrics.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/Metrics.apt.vm index 55e532df9fc..14cc712f0e4 100644 --- a/hadoop-common-project/hadoop-common/src/site/apt/Metrics.apt.vm +++ b/hadoop-common-project/hadoop-common/src/site/apt/Metrics.apt.vm @@ -86,6 +86,14 @@ jvm context *-------------------------------------+--------------------------------------+ |<<>> | Total number of INFO logs *-------------------------------------+--------------------------------------+ +|<<>> | Number of times that the GC warn + | threshold is exceeded +*-------------------------------------+--------------------------------------+ +|<<>> | Number of times that the GC info + | threshold is exceeded +*-------------------------------------+--------------------------------------+ +|<<>> | Total GC extra sleep time in msec +*-------------------------------------+--------------------------------------+ rpc context diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/source/TestJvmMetrics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/source/TestJvmMetrics.java index 1fc91c39bb1..3cb3384d839 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/source/TestJvmMetrics.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/source/TestJvmMetrics.java @@ -19,18 +19,25 @@ package org.apache.hadoop.metrics2.source; import org.junit.Test; + import static org.mockito.Mockito.*; import static org.apache.hadoop.test.MetricsAsserts.*; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.metrics2.MetricsCollector; import org.apache.hadoop.metrics2.MetricsRecordBuilder; +import org.apache.hadoop.util.JvmPauseMonitor; + import static org.apache.hadoop.metrics2.source.JvmMetricsInfo.*; import static org.apache.hadoop.metrics2.impl.MsInfo.*; public class TestJvmMetrics { @Test public void testPresence() { - MetricsRecordBuilder rb = getMetrics(new JvmMetrics("test", "test")); + JvmPauseMonitor pauseMonitor = new JvmPauseMonitor(new Configuration()); + JvmMetrics jvmMetrics = new JvmMetrics("test", "test"); + jvmMetrics.setPauseMonitor(pauseMonitor); + MetricsRecordBuilder rb = getMetrics(jvmMetrics); MetricsCollector mc = rb.parent(); verify(mc).addRecord(JvmMetrics); diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index baf65464ef5..dae38c218c8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -453,6 +453,9 @@ Release 2.5.0 - UNRELEASED HDFS-6480. Move waitForReady() from FSDirectory to FSNamesystem. (wheat9) + HDFS-6403. Add metrics for log warnings reported by JVM pauses. (Yongjun + Zhang via atm) + OPTIMIZATIONS HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 292259ad197..5797c00c402 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -778,7 +778,8 @@ void startDataNode(Configuration conf, initIpcServer(conf); metrics = DataNodeMetrics.create(conf, getDisplayName()); - + metrics.getJvmMetrics().setPauseMonitor(pauseMonitor); + blockPoolManager = new BlockPoolManager(this); blockPoolManager.refreshNamenodes(conf); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java index 9601bcf0c67..b536e7e954f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java @@ -90,13 +90,15 @@ public class DataNodeMetrics { final MutableQuantiles[] sendDataPacketBlockedOnNetworkNanosQuantiles; @Metric MutableRate sendDataPacketTransferNanos; final MutableQuantiles[] sendDataPacketTransferNanosQuantiles; - final MetricsRegistry registry = new MetricsRegistry("datanode"); final String name; - - public DataNodeMetrics(String name, String sessionId, int[] intervals) { + JvmMetrics jvmMetrics = null; + + public DataNodeMetrics(String name, String sessionId, int[] intervals, + final JvmMetrics jvmMetrics) { this.name = name; + this.jvmMetrics = jvmMetrics; registry.tag(SessionId, sessionId); final int len = intervals.length; @@ -131,7 +133,7 @@ public DataNodeMetrics(String name, String sessionId, int[] intervals) { public static DataNodeMetrics create(Configuration conf, String dnName) { String sessionId = conf.get(DFSConfigKeys.DFS_METRICS_SESSION_ID_KEY); MetricsSystem ms = DefaultMetricsSystem.instance(); - JvmMetrics.create("DataNode", sessionId, ms); + JvmMetrics jm = JvmMetrics.create("DataNode", sessionId, ms); String name = "DataNodeActivity-"+ (dnName.isEmpty() ? "UndefinedDataNodeName"+ DFSUtil.getRandom().nextInt() : dnName.replace(':', '-')); @@ -141,11 +143,15 @@ public static DataNodeMetrics create(Configuration conf, String dnName) { conf.getInts(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY); return ms.register(name, null, new DataNodeMetrics(name, sessionId, - intervals)); + intervals, jm)); } public String name() { return name; } + public JvmMetrics getJvmMetrics() { + return jvmMetrics; + } + public void addHeartbeat(long latency) { heartbeats.add(latency); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index 9cdad26eb24..9747f9cddac 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -598,7 +598,8 @@ protected void initialize(Configuration conf) throws IOException { pauseMonitor = new JvmPauseMonitor(conf); pauseMonitor.start(); - + metrics.getJvmMetrics().setPauseMonitor(pauseMonitor); + startCommonServices(conf); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java index 52ad185cdf6..42942dc51bc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java @@ -98,7 +98,11 @@ public class NameNodeMetrics { @Metric("GetImageServlet putImage") MutableRate putImage; - NameNodeMetrics(String processName, String sessionId, int[] intervals) { + JvmMetrics jvmMetrics = null; + + NameNodeMetrics(String processName, String sessionId, int[] intervals, + final JvmMetrics jvmMetrics) { + this.jvmMetrics = jvmMetrics; registry.tag(ProcessName, processName).tag(SessionId, sessionId); final int len = intervals.length; @@ -124,14 +128,19 @@ public static NameNodeMetrics create(Configuration conf, NamenodeRole r) { String sessionId = conf.get(DFSConfigKeys.DFS_METRICS_SESSION_ID_KEY); String processName = r.toString(); MetricsSystem ms = DefaultMetricsSystem.instance(); - JvmMetrics.create(processName, sessionId, ms); + JvmMetrics jm = JvmMetrics.create(processName, sessionId, ms); // Percentile measurement is off by default, by watching no intervals int[] intervals = conf.getInts(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY); - return ms.register(new NameNodeMetrics(processName, sessionId, intervals)); + return ms.register(new NameNodeMetrics(processName, sessionId, + intervals, jm)); } + public JvmMetrics getJvmMetrics() { + return jvmMetrics; + } + public void shutdown() { DefaultMetricsSystem.shutdown(); } From e74d99b81e57262e717d012bdb3ed793eebfb45b Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Fri, 20 Jun 2014 05:53:41 +0000 Subject: [PATCH 006/112] HADOOP-10279. Create multiplexer, a requirement for the fair queue. (Contributed by Chris Li) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1604090 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../ipc/WeightedRoundRobinMultiplexer.java | 148 ++++++++++++++++++ .../TestWeightedRoundRobinMultiplexer.java | 142 +++++++++++++++++ 3 files changed, 293 insertions(+) create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WeightedRoundRobinMultiplexer.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestWeightedRoundRobinMultiplexer.java diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index fe0ef94e0af..d4067966169 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -442,6 +442,9 @@ Release 2.5.0 - UNRELEASED HADOOP-10557. FsShell -cp -pa option for preserving extended ACLs. (Akira Ajisaka via cnauroth) + HADOOP-10279. Create multiplexer, a requirement for the fair queue. + (Chris Li via Arpit Agarwal) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WeightedRoundRobinMultiplexer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WeightedRoundRobinMultiplexer.java new file mode 100644 index 00000000000..497ca757461 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/WeightedRoundRobinMultiplexer.java @@ -0,0 +1,148 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ipc; + +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; + +/** + * Determines which queue to start reading from, occasionally drawing from + * low-priority queues in order to prevent starvation. Given the pull pattern + * [9, 4, 1] for 3 queues: + * + * The cycle is (a minimum of) 9+4+1=14 reads. + * Queue 0 is read (at least) 9 times + * Queue 1 is read (at least) 4 times + * Queue 2 is read (at least) 1 time + * Repeat + * + * There may be more reads than the minimum due to race conditions. This is + * allowed by design for performance reasons. + */ +public class WeightedRoundRobinMultiplexer { + // Config keys + public static final String IPC_CALLQUEUE_WRRMUX_WEIGHTS_KEY = + "faircallqueue.multiplexer.weights"; + + public static final Log LOG = + LogFactory.getLog(WeightedRoundRobinMultiplexer.class); + + private final int numQueues; // The number of queues under our provisioning + + private final AtomicInteger currentQueueIndex; // Current queue we're serving + private final AtomicInteger requestsLeft; // Number of requests left for this queue + + private int[] queueWeights; // The weights for each queue + + public WeightedRoundRobinMultiplexer(int aNumQueues, String ns, + Configuration conf) { + if (aNumQueues <= 0) { + throw new IllegalArgumentException("Requested queues (" + aNumQueues + + ") must be greater than zero."); + } + + this.numQueues = aNumQueues; + this.queueWeights = conf.getInts(ns + "." + + IPC_CALLQUEUE_WRRMUX_WEIGHTS_KEY); + + if (this.queueWeights.length == 0) { + this.queueWeights = getDefaultQueueWeights(this.numQueues); + } else if (this.queueWeights.length != this.numQueues) { + throw new IllegalArgumentException(ns + "." + + IPC_CALLQUEUE_WRRMUX_WEIGHTS_KEY + " must specify exactly " + + this.numQueues + " weights: one for each priority level."); + } + + this.currentQueueIndex = new AtomicInteger(0); + this.requestsLeft = new AtomicInteger(this.queueWeights[0]); + + LOG.info("WeightedRoundRobinMultiplexer is being used."); + } + + /** + * Creates default weights for each queue. The weights are 2^N. + */ + private int[] getDefaultQueueWeights(int aNumQueues) { + int[] weights = new int[aNumQueues]; + + int weight = 1; // Start low + for(int i = aNumQueues - 1; i >= 0; i--) { // Start at lowest queue + weights[i] = weight; + weight *= 2; // Double every iteration + } + return weights; + } + + /** + * Move to the next queue. + */ + private void moveToNextQueue() { + int thisIdx = this.currentQueueIndex.get(); + + // Wrap to fit in our bounds + int nextIdx = (thisIdx + 1) % this.numQueues; + + // Set to next index: once this is called, requests will start being + // drawn from nextIdx, but requestsLeft will continue to decrement into + // the negatives + this.currentQueueIndex.set(nextIdx); + + // Finally, reset requestsLeft. This will enable moveToNextQueue to be + // called again, for the new currentQueueIndex + this.requestsLeft.set(this.queueWeights[nextIdx]); + } + + /** + * Advances the index, which will change the current index + * if called enough times. + */ + private void advanceIndex() { + // Since we did read, we should decrement + int requestsLeftVal = this.requestsLeft.decrementAndGet(); + + // Strict compare with zero (instead of inequality) so that if another + // thread decrements requestsLeft, only one thread will be responsible + // for advancing currentQueueIndex + if (requestsLeftVal == 0) { + // This is guaranteed to be called exactly once per currentQueueIndex + this.moveToNextQueue(); + } + } + + /** + * Gets the current index. Should be accompanied by a call to + * advanceIndex at some point. + */ + private int getCurrentIndex() { + return this.currentQueueIndex.get(); + } + + /** + * Use the mux by getting and advancing index. + */ + public int getAndAdvanceCurrentIndex() { + int idx = this.getCurrentIndex(); + this.advanceIndex(); + return idx; + } + +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestWeightedRoundRobinMultiplexer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestWeightedRoundRobinMultiplexer.java new file mode 100644 index 00000000000..642817617e5 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestWeightedRoundRobinMultiplexer.java @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ipc; + +import static org.junit.Assert.assertEquals; +import org.junit.Test; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; + +import static org.apache.hadoop.ipc.WeightedRoundRobinMultiplexer.IPC_CALLQUEUE_WRRMUX_WEIGHTS_KEY; + +public class TestWeightedRoundRobinMultiplexer { + public static final Log LOG = LogFactory.getLog(TestWeightedRoundRobinMultiplexer.class); + + private WeightedRoundRobinMultiplexer mux; + + @Test(expected=IllegalArgumentException.class) + public void testInstantiateNegativeMux() { + mux = new WeightedRoundRobinMultiplexer(-1, "", new Configuration()); + } + + @Test(expected=IllegalArgumentException.class) + public void testInstantiateZeroMux() { + mux = new WeightedRoundRobinMultiplexer(0, "", new Configuration()); + } + + @Test(expected=IllegalArgumentException.class) + public void testInstantiateIllegalMux() { + Configuration conf = new Configuration(); + conf.setStrings("namespace." + IPC_CALLQUEUE_WRRMUX_WEIGHTS_KEY, + "1", "2", "3"); + + // ask for 3 weights with 2 queues + mux = new WeightedRoundRobinMultiplexer(2, "namespace", conf); + } + + @Test + public void testLegalInstantiation() { + Configuration conf = new Configuration(); + conf.setStrings("namespace." + IPC_CALLQUEUE_WRRMUX_WEIGHTS_KEY, + "1", "2", "3"); + + // ask for 3 weights with 3 queues + mux = new WeightedRoundRobinMultiplexer(3, "namespace.", conf); + } + + @Test + public void testDefaultPattern() { + // Mux of size 1: 0 0 0 0 0, etc + mux = new WeightedRoundRobinMultiplexer(1, "", new Configuration()); + for(int i = 0; i < 10; i++) { + assertEquals(mux.getAndAdvanceCurrentIndex(), 0); + } + + // Mux of size 2: 0 0 1 0 0 1 0 0 1, etc + mux = new WeightedRoundRobinMultiplexer(2, "", new Configuration()); + assertEquals(mux.getAndAdvanceCurrentIndex(), 0); + assertEquals(mux.getAndAdvanceCurrentIndex(), 0); + assertEquals(mux.getAndAdvanceCurrentIndex(), 1); + assertEquals(mux.getAndAdvanceCurrentIndex(), 0); + assertEquals(mux.getAndAdvanceCurrentIndex(), 0); + assertEquals(mux.getAndAdvanceCurrentIndex(), 1); + + // Size 3: 4x0 2x1 1x2, etc + mux = new WeightedRoundRobinMultiplexer(3, "", new Configuration()); + assertEquals(mux.getAndAdvanceCurrentIndex(), 0); + assertEquals(mux.getAndAdvanceCurrentIndex(), 0); + assertEquals(mux.getAndAdvanceCurrentIndex(), 0); + assertEquals(mux.getAndAdvanceCurrentIndex(), 0); + assertEquals(mux.getAndAdvanceCurrentIndex(), 1); + assertEquals(mux.getAndAdvanceCurrentIndex(), 1); + assertEquals(mux.getAndAdvanceCurrentIndex(), 2); + assertEquals(mux.getAndAdvanceCurrentIndex(), 0); + + // Size 4: 8x0 4x1 2x2 1x3 + mux = new WeightedRoundRobinMultiplexer(4, "", new Configuration()); + assertEquals(mux.getAndAdvanceCurrentIndex(), 0); + assertEquals(mux.getAndAdvanceCurrentIndex(), 0); + assertEquals(mux.getAndAdvanceCurrentIndex(), 0); + assertEquals(mux.getAndAdvanceCurrentIndex(), 0); + assertEquals(mux.getAndAdvanceCurrentIndex(), 0); + assertEquals(mux.getAndAdvanceCurrentIndex(), 0); + assertEquals(mux.getAndAdvanceCurrentIndex(), 0); + assertEquals(mux.getAndAdvanceCurrentIndex(), 0); + assertEquals(mux.getAndAdvanceCurrentIndex(), 1); + assertEquals(mux.getAndAdvanceCurrentIndex(), 1); + assertEquals(mux.getAndAdvanceCurrentIndex(), 1); + assertEquals(mux.getAndAdvanceCurrentIndex(), 1); + assertEquals(mux.getAndAdvanceCurrentIndex(), 2); + assertEquals(mux.getAndAdvanceCurrentIndex(), 2); + assertEquals(mux.getAndAdvanceCurrentIndex(), 3); + assertEquals(mux.getAndAdvanceCurrentIndex(), 0); + } + + @Test + public void testCustomPattern() { + // 1x0 1x1 + Configuration conf = new Configuration(); + conf.setStrings("test.custom." + IPC_CALLQUEUE_WRRMUX_WEIGHTS_KEY, + "1", "1"); + + mux = new WeightedRoundRobinMultiplexer(2, "test.custom", conf); + assertEquals(mux.getAndAdvanceCurrentIndex(), 0); + assertEquals(mux.getAndAdvanceCurrentIndex(), 1); + assertEquals(mux.getAndAdvanceCurrentIndex(), 0); + assertEquals(mux.getAndAdvanceCurrentIndex(), 1); + + // 1x0 3x1 2x2 + conf.setStrings("test.custom." + IPC_CALLQUEUE_WRRMUX_WEIGHTS_KEY, + "1", "3", "2"); + + mux = new WeightedRoundRobinMultiplexer(3, "test.custom", conf); + + for(int i = 0; i < 5; i++) { + assertEquals(mux.getAndAdvanceCurrentIndex(), 0); + assertEquals(mux.getAndAdvanceCurrentIndex(), 1); + assertEquals(mux.getAndAdvanceCurrentIndex(), 1); + assertEquals(mux.getAndAdvanceCurrentIndex(), 1); + assertEquals(mux.getAndAdvanceCurrentIndex(), 2); + assertEquals(mux.getAndAdvanceCurrentIndex(), 2); + } // Ensure pattern repeats + + } +} \ No newline at end of file From f8041b05404dce7088d73f5befc500cb2786c9bd Mon Sep 17 00:00:00 2001 From: Chris Nauroth Date: Fri, 20 Jun 2014 18:24:03 +0000 Subject: [PATCH 007/112] HADOOP-9559. When metrics system is restarted MBean names get incorrectly flagged as dupes. Contributed by Mostafa Elhemali and Mike Liddell. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1604225 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 +++ .../metrics2/impl/MetricsSourceAdapter.java | 7 +++++++ .../metrics2/impl/MetricsSystemImpl.java | 6 ++++++ .../metrics2/lib/DefaultMetricsSystem.java | 9 +++++++++ .../apache/hadoop/metrics2/util/MBeans.java | 1 + .../metrics2/impl/TestMetricsSystemImpl.java | 18 ++++++++++++++++++ 6 files changed, 44 insertions(+) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index d4067966169..fe80214f79a 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -582,6 +582,9 @@ Release 2.5.0 - UNRELEASED HADOOP-10716. Cannot use more than 1 har filesystem. (Rushabh Shah via cnauroth) + HADOOP-9559. When metrics system is restarted MBean names get incorrectly + flagged as dupes. (Mostafa Elhemali and Mike Liddell via cnauroth) + BREAKDOWN OF HADOOP-10514 SUBTASKS AND RELATED JIRAS HADOOP-10520. Extended attributes definition and FileSystem APIs for diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSourceAdapter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSourceAdapter.java index 23c4492663b..cf11e6db14e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSourceAdapter.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSourceAdapter.java @@ -30,6 +30,7 @@ import javax.management.ReflectionException; import static com.google.common.base.Preconditions.*; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Maps; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -226,7 +227,13 @@ synchronized void stopMBeans() { mbeanName = null; } } + + @VisibleForTesting + ObjectName getMBeanName() { + return mbeanName; + } + private void updateInfoCache() { LOG.debug("Updating info cache..."); infoCache = infoBuilder.reset(lastRecs).get(); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java index e386ce608f0..cf2dda4e380 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java @@ -32,6 +32,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.annotations.VisibleForTesting; import java.util.Locale; import static com.google.common.base.Preconditions.*; @@ -573,6 +574,11 @@ public MetricsSource getSource(String name) { return allSources.get(name); } + @VisibleForTesting + MetricsSourceAdapter getSourceAdapter(String name) { + return sources.get(name); + } + private InitMode initMode() { LOG.debug("from system property: "+ System.getProperty(MS_INIT_MODE_KEY)); LOG.debug("from environment variable: "+ System.getenv(MS_INIT_MODE_KEY)); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/DefaultMetricsSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/DefaultMetricsSystem.java index be785b98544..c761b583937 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/DefaultMetricsSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/DefaultMetricsSystem.java @@ -110,6 +110,11 @@ public static ObjectName newMBeanName(String name) { return INSTANCE.newObjectName(name); } + @InterfaceAudience.Private + public static void removeMBeanName(ObjectName name) { + INSTANCE.removeObjectName(name.toString()); + } + @InterfaceAudience.Private public static String sourceName(String name, boolean dupOK) { return INSTANCE.newSourceName(name, dupOK); @@ -126,6 +131,10 @@ synchronized ObjectName newObjectName(String name) { } } + synchronized void removeObjectName(String name) { + mBeanNames.map.remove(name); + } + synchronized String newSourceName(String name, boolean dupOK) { if (sourceNames.map.containsKey(name)) { if (dupOK) { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/MBeans.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/MBeans.java index 8f2c01209ce..aec9dad40c2 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/MBeans.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/util/MBeans.java @@ -84,6 +84,7 @@ static public void unregister(ObjectName mbeanName) { } catch (Exception e) { LOG.warn("Error unregistering "+ mbeanName, e); } + DefaultMetricsSystem.removeMBeanName(mbeanName); } static private ObjectName getMBeanName(String serviceName, String nameName) { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java index 49a54af5998..564214bba65 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java @@ -360,6 +360,24 @@ public void flush() { ms.register(ts); } + @Test public void testStartStopStart() { + DefaultMetricsSystem.shutdown(); // Clear pre-existing source names. + MetricsSystemImpl ms = new MetricsSystemImpl("test"); + TestSource ts = new TestSource("ts"); + ms.start(); + ms.register("ts", "", ts); + MetricsSourceAdapter sa = ms.getSourceAdapter("ts"); + assertNotNull(sa); + assertNotNull(sa.getMBeanName()); + ms.stop(); + ms.shutdown(); + ms.start(); + sa = ms.getSourceAdapter("ts"); + assertNotNull(sa); + assertNotNull(sa.getMBeanName()); + ms.stop(); + ms.shutdown(); + } private void checkMetricsRecords(List recs) { LOG.debug(recs); From d9eb18bb2e6c25ecc3acaaa2f7e53aff1d795edd Mon Sep 17 00:00:00 2001 From: Jing Zhao Date: Fri, 20 Jun 2014 18:25:09 +0000 Subject: [PATCH 008/112] HDFS-6535. HDFS quota update is wrong when file is appended. Contributed by George Wong. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1604226 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../hdfs/server/namenode/FSNamesystem.java | 2 +- .../namenode/TestDiskspaceQuotaUpdate.java | 64 ++++++++++++++++--- 3 files changed, 59 insertions(+), 10 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index dae38c218c8..c01326be28e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -679,6 +679,9 @@ Release 2.5.0 - UNRELEASED HDFS-6549. Add support for accessing the NFS gateway from the AIX NFS client. (atm) + HDFS-6535. HDFS quota update is wrong when file is appended. (George Wong + via jing9) + BREAKDOWN OF HDFS-2006 SUBTASKS AND RELATED JIRAS HDFS-6299. Protobuf for XAttr and client-side implementation. (Yi Liu via umamahesh) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index ad7b4d61285..af436182abd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -2512,7 +2512,7 @@ LocatedBlock prepareFileForWrite(String src, INodeFile file, if (ret != null) { // update the quota: use the preferred block size for UC block final long diff = file.getPreferredBlockSize() - ret.getBlockSize(); - dir.updateSpaceConsumed(src, 0, diff); + dir.updateSpaceConsumed(src, 0, diff * file.getBlockReplication()); } if (writeToEditLog) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java index 1c770b141af..fad88070ad5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java @@ -18,10 +18,12 @@ package org.apache.hadoop.hdfs.server.namenode; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import java.util.EnumSet; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSConfigKeys; @@ -36,7 +38,9 @@ public class TestDiskspaceQuotaUpdate { private static final int BLOCKSIZE = 1024; - private static final short REPLICATION = 1; + private static final short REPLICATION = 4; + static final long seed = 0L; + private static final Path dir = new Path("/TestQuotaUpdate"); private Configuration conf; private MiniDFSCluster cluster; @@ -62,42 +66,84 @@ public void tearDown() throws Exception { } } + /** + * Test if the quota can be correctly updated for create file + */ + @Test (timeout=60000) + public void testQuotaUpdateWithFileCreate() throws Exception { + final Path foo = new Path(dir, "foo"); + Path createdFile = new Path(foo, "created_file.data"); + dfs.mkdirs(foo); + dfs.setQuota(foo, Long.MAX_VALUE-1, Long.MAX_VALUE-1); + long fileLen = BLOCKSIZE * 2 + BLOCKSIZE / 2; + DFSTestUtil.createFile(dfs, createdFile, BLOCKSIZE / 16, + fileLen, BLOCKSIZE, REPLICATION, seed); + INode fnode = fsdir.getINode4Write(foo.toString()); + assertTrue(fnode.isDirectory()); + assertTrue(fnode.isQuotaSet()); + Quota.Counts cnt = fnode.asDirectory().getDirectoryWithQuotaFeature() + .getSpaceConsumed(); + assertEquals(2, cnt.get(Quota.NAMESPACE)); + assertEquals(fileLen * REPLICATION, cnt.get(Quota.DISKSPACE)); + } + /** * Test if the quota can be correctly updated for append */ - @Test + @Test (timeout=60000) public void testUpdateQuotaForAppend() throws Exception { - final Path foo = new Path("/foo"); + final Path foo = new Path(dir ,"foo"); final Path bar = new Path(foo, "bar"); - DFSTestUtil.createFile(dfs, bar, BLOCKSIZE, REPLICATION, 0L); + long currentFileLen = BLOCKSIZE; + DFSTestUtil.createFile(dfs, bar, currentFileLen, REPLICATION, seed); dfs.setQuota(foo, Long.MAX_VALUE-1, Long.MAX_VALUE-1); - // append half of the block data + // append half of the block data, the previous file length is at block + // boundary DFSTestUtil.appendFile(dfs, bar, BLOCKSIZE / 2); + currentFileLen += (BLOCKSIZE / 2); INodeDirectory fooNode = fsdir.getINode4Write(foo.toString()).asDirectory(); + assertTrue(fooNode.isQuotaSet()); Quota.Counts quota = fooNode.getDirectoryWithQuotaFeature() .getSpaceConsumed(); long ns = quota.get(Quota.NAMESPACE); long ds = quota.get(Quota.DISKSPACE); assertEquals(2, ns); // foo and bar - assertEquals((BLOCKSIZE + BLOCKSIZE / 2) * REPLICATION, ds); + assertEquals(currentFileLen * REPLICATION, ds); + ContentSummary c = dfs.getContentSummary(foo); + assertEquals(c.getSpaceConsumed(), ds); - // append another block + // append another block, the previous file length is not at block boundary DFSTestUtil.appendFile(dfs, bar, BLOCKSIZE); + currentFileLen += BLOCKSIZE; quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed(); ns = quota.get(Quota.NAMESPACE); ds = quota.get(Quota.DISKSPACE); assertEquals(2, ns); // foo and bar - assertEquals((BLOCKSIZE * 2 + BLOCKSIZE / 2) * REPLICATION, ds); + assertEquals(currentFileLen * REPLICATION, ds); + c = dfs.getContentSummary(foo); + assertEquals(c.getSpaceConsumed(), ds); + + // append several blocks + DFSTestUtil.appendFile(dfs, bar, BLOCKSIZE * 3 + BLOCKSIZE / 8); + currentFileLen += (BLOCKSIZE * 3 + BLOCKSIZE / 8); + + quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed(); + ns = quota.get(Quota.NAMESPACE); + ds = quota.get(Quota.DISKSPACE); + assertEquals(2, ns); // foo and bar + assertEquals(currentFileLen * REPLICATION, ds); + c = dfs.getContentSummary(foo); + assertEquals(c.getSpaceConsumed(), ds); } /** * Test if the quota can be correctly updated when file length is updated * through fsync */ - @Test + @Test (timeout=60000) public void testUpdateQuotaForFSync() throws Exception { final Path foo = new Path("/foo"); final Path bar = new Path(foo, "bar"); From 1355ff613260566f860aefa94690321c34c40d16 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Fri, 20 Jun 2014 18:28:21 +0000 Subject: [PATCH 009/112] MAPREDUCE-5830. Added back the private API HostUtil.getTaskLogUrl(..) for binary compatibility with older clients like Hive 0.13. Contributed by Akira Ajisaka. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1604230 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 4 ++++ .../apache/hadoop/mapreduce/util/HostUtil.java | 15 +++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 8feab76d100..1220895908b 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -281,6 +281,10 @@ Release 2.4.1 - 2014-06-23 IMPROVEMENTS + MAPREDUCE-5830. Added back the private API HostUtil.getTaskLogUrl(..) for + binary compatibility with older clients like Hive 0.13. (Akira Ajisaka via + vinodkv) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java index e131fc8933a..ad279ee0748 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/HostUtil.java @@ -38,6 +38,21 @@ public static String getTaskLogUrl(String scheme, String taskTrackerHostName, httpPort + "/tasklog?attemptid=" + taskAttemptID); } + /** + * Always throws {@link RuntimeException} because this method is not + * supposed to be called at runtime. This method is only for keeping + * binary compatibility with Hive 0.13. MAPREDUCE-5830 for the details. + * @deprecated Use {@link #getTaskLogUrl(String, String, String, String)} + * to construct the taskLogUrl. + */ + @Deprecated + public static String getTaskLogUrl(String taskTrackerHostName, + String httpPort, String taskAttemptID) { + throw new RuntimeException( + "This method is not supposed to be called at runtime. " + + "Use HostUtil.getTaskLogUrl(String, String, String, String) instead."); + } + public static String convertTrackerNameToHostName(String trackerName) { // Ugly! // Convert the trackerName to its host name From 8300b9fb385b47672d98ea62ab291991424f3cce Mon Sep 17 00:00:00 2001 From: Chris Nauroth Date: Fri, 20 Jun 2014 18:30:49 +0000 Subject: [PATCH 010/112] HADOOP-10689. InputStream is not closed in AzureNativeFileSystemStore#retrieve(). Contributed by Chen He. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1604233 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index fe80214f79a..528f5e5a255 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -585,6 +585,9 @@ Release 2.5.0 - UNRELEASED HADOOP-9559. When metrics system is restarted MBean names get incorrectly flagged as dupes. (Mostafa Elhemali and Mike Liddell via cnauroth) + HADOOP-10689. InputStream is not closed in + AzureNativeFileSystemStore#retrieve(). (Chen He via cnauroth) + BREAKDOWN OF HADOOP-10514 SUBTASKS AND RELATED JIRAS HADOOP-10520. Extended attributes definition and FileSystem APIs for diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java index 9dcaddc2315..c5b9afe6d80 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java @@ -1723,7 +1723,7 @@ public DataInputStream retrieve(String key, long startByteOffset) inDataStream.close(); } if(in != null){ - inDataStream.close(); + in.close(); } throw e; } From 61bf9f779919206296d2ce84a8a6b2d912709a59 Mon Sep 17 00:00:00 2001 From: Chris Nauroth Date: Fri, 20 Jun 2014 18:36:11 +0000 Subject: [PATCH 011/112] HADOOP-10690. Lack of synchronization on access to InputStream in NativeAzureFileSystem#NativeAzureFsInputStream#close(). Contributed by Chen He. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1604236 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 4 ++++ .../org/apache/hadoop/fs/azure/NativeAzureFileSystem.java | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 528f5e5a255..3adee445e54 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -588,6 +588,10 @@ Release 2.5.0 - UNRELEASED HADOOP-10689. InputStream is not closed in AzureNativeFileSystemStore#retrieve(). (Chen He via cnauroth) + HADOOP-10690. Lack of synchronization on access to InputStream in + NativeAzureFileSystem#NativeAzureFsInputStream#close(). + (Chen He via cnauroth) + BREAKDOWN OF HADOOP-10514 SUBTASKS AND RELATED JIRAS HADOOP-10520. Extended attributes definition and FileSystem APIs for diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java index 30e6b3091e0..87dc9787f0e 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java @@ -193,7 +193,7 @@ public synchronized int read(byte[] b, int off, int len) throws IOException { } @Override - public void close() throws IOException { + public synchronized void close() throws IOException { in.close(); } From 9ca79e8d327e95845ef9794396afd43a52bc3d40 Mon Sep 17 00:00:00 2001 From: Haohui Mai Date: Fri, 20 Jun 2014 18:54:19 +0000 Subject: [PATCH 012/112] HDFS-6557. Move the reference of fsimage to FSNamesystem. Contributed by Haohui Mai. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1604242 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../hdfs/server/namenode/FSDirectory.java | 8 +--- .../hadoop/hdfs/server/namenode/FSImage.java | 2 - .../hdfs/server/namenode/FSNamesystem.java | 44 ++++++++++--------- .../hadoop/hdfs/server/namenode/NameNode.java | 4 +- .../hdfs/server/namenode/NameNodeAdapter.java | 6 ++- .../namenode/TestFSPermissionChecker.java | 3 +- .../server/namenode/TestSaveNamespace.java | 16 ++++--- 8 files changed, 43 insertions(+), 42 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index c01326be28e..44020ead059 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -456,6 +456,8 @@ Release 2.5.0 - UNRELEASED HDFS-6403. Add metrics for log warnings reported by JVM pauses. (Yongjun Zhang via atm) + HDFS-6557. Move the reference of fsimage to FSNamesystem. (wheat9) + OPTIMIZATIONS HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index da8d11e9aba..d00372957f9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@ -117,7 +117,6 @@ private static INodeDirectorySnapshottable createRoot(FSNamesystem namesystem) { public final static byte[] DOT_INODES = DFSUtil.string2Bytes(DOT_INODES_STRING); INodeDirectory rootDir; - FSImage fsImage; private final FSNamesystem namesystem; private volatile boolean skipQuotaCheck = false; //skip while consuming edits private final int maxComponentLength; @@ -170,11 +169,10 @@ public int getWriteHoldCount() { */ private final NameCache nameCache; - FSDirectory(FSImage fsImage, FSNamesystem ns, Configuration conf) { + FSDirectory(FSNamesystem ns, Configuration conf) { this.dirLock = new ReentrantReadWriteLock(true); // fair rootDir = createRoot(ns); inodeMap = INodeMap.newInstance(rootDir); - this.fsImage = fsImage; int configuredLimit = conf.getInt( DFSConfigKeys.DFS_LIST_LIMIT, DFSConfigKeys.DFS_LIST_LIMIT_DEFAULT); this.lsLimit = configuredLimit>0 ? @@ -231,9 +229,7 @@ public INodeDirectory getRoot() { * Shutdown the filestore */ @Override - public void close() throws IOException { - fsImage.close(); - } + public void close() throws IOException {} void markNameCacheInitialized() { writeLock(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java index b163579f0c8..d99985e5dc9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java @@ -495,7 +495,6 @@ void doImportCheckpoint(FSNamesystem target) throws IOException { FSImage realImage = target.getFSImage(); FSImage ckptImage = new FSImage(conf, checkpointDirs, checkpointEditsDirs); - target.dir.fsImage = ckptImage; // load from the checkpoint dirs try { ckptImage.recoverTransitionRead(StartupOption.REGULAR, target, null); @@ -507,7 +506,6 @@ void doImportCheckpoint(FSNamesystem target) throws IOException { realImage.getEditLog().setNextTxId(ckptImage.getEditLog().getLastWrittenTxId()+1); realImage.initEditLog(StartupOption.IMPORT); - target.dir.fsImage = realImage; realImage.getStorage().setBlockPoolID(ckptImage.getBlockPoolID()); // and save it but keep the same checkpointTime diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index af436182abd..a66578f4dfb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -517,6 +517,9 @@ private void logAuditEvent(boolean succeeded, private volatile boolean imageLoaded = false; private final Condition cond; + + private final FSImage fsImage; + /** * Notify that loading of this FSDirectory is complete, and * it is imageLoaded for use @@ -738,6 +741,7 @@ static FSNamesystem loadFromDisk(Configuration conf) throws IOException { LOG.info("fsLock is fair:" + fair); fsLock = new FSNamesystemLock(fair); cond = fsLock.writeLock().newCondition(); + this.fsImage = fsImage; try { resourceRecheckInterval = conf.getLong( DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_KEY, @@ -827,7 +831,7 @@ static FSNamesystem loadFromDisk(Configuration conf) throws IOException { DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_DEFAULT); this.dtSecretManager = createDelegationTokenSecretManager(conf); - this.dir = new FSDirectory(fsImage, this, conf); + this.dir = new FSDirectory(this, conf); this.snapshotManager = new SnapshotManager(dir); this.cacheManager = new CacheManager(this, conf, blockManager); this.safeMode = new SafeModeInfo(conf); @@ -1055,7 +1059,7 @@ void startActiveServices() throws IOException { LOG.info("Starting services required for active state"); writeLock(); try { - FSEditLog editLog = dir.fsImage.getEditLog(); + FSEditLog editLog = getFSImage().getEditLog(); if (!editLog.isOpenForWrite()) { // During startup, we're already open for write during initialization. @@ -1084,12 +1088,12 @@ void startActiveServices() throws IOException { metaSaveAsString()); } - long nextTxId = dir.fsImage.getLastAppliedTxId() + 1; + long nextTxId = getFSImage().getLastAppliedTxId() + 1; LOG.info("Will take over writing edit logs at txnid " + nextTxId); editLog.setNextTxId(nextTxId); - dir.fsImage.editLog.openForWrite(); + getFSImage().editLog.openForWrite(); } // Enable quota checks. @@ -1164,13 +1168,13 @@ void stopActiveServices() { ((NameNodeEditLogRoller)nnEditLogRoller.getRunnable()).stop(); nnEditLogRoller.interrupt(); } - if (dir != null && dir.fsImage != null) { - if (dir.fsImage.editLog != null) { - dir.fsImage.editLog.close(); + if (dir != null && getFSImage() != null) { + if (getFSImage().editLog != null) { + getFSImage().editLog.close(); } // Update the fsimage with the last txid that we wrote // so that the tailer starts from the right spot. - dir.fsImage.updateLastAppliedTxIdFromWritten(); + getFSImage().updateLastAppliedTxIdFromWritten(); } if (cacheManager != null) { cacheManager.stopMonitorThread(); @@ -1193,9 +1197,9 @@ void stopActiveServices() { */ void startStandbyServices(final Configuration conf) throws IOException { LOG.info("Starting services required for standby state"); - if (!dir.fsImage.editLog.isOpenForRead()) { + if (!getFSImage().editLog.isOpenForRead()) { // During startup, we're already open for read. - dir.fsImage.editLog.initSharedJournalsForRead(); + getFSImage().editLog.initSharedJournalsForRead(); } blockManager.setPostponeBlocksFromFuture(true); @@ -1242,8 +1246,8 @@ void stopStandbyServices() throws IOException { if (editLogTailer != null) { editLogTailer.stop(); } - if (dir != null && dir.fsImage != null && dir.fsImage.editLog != null) { - dir.fsImage.editLog.close(); + if (dir != null && getFSImage() != null && getFSImage().editLog != null) { + getFSImage().editLog.close(); } } @@ -1486,9 +1490,9 @@ NamespaceInfo getNamespaceInfo() { * Version of @see #getNamespaceInfo() that is not protected by a lock. */ NamespaceInfo unprotectedGetNamespaceInfo() { - return new NamespaceInfo(dir.fsImage.getStorage().getNamespaceID(), + return new NamespaceInfo(getFSImage().getStorage().getNamespaceID(), getClusterId(), getBlockPoolId(), - dir.fsImage.getStorage().getCTime()); + getFSImage().getStorage().getCTime()); } /** @@ -1506,12 +1510,10 @@ void close() { try { stopActiveServices(); stopStandbyServices(); - if (dir != null) { - dir.close(); - } } catch (IOException ie) { - LOG.error("Error closing FSDirectory", ie); + } finally { IOUtils.cleanup(LOG, dir); + IOUtils.cleanup(LOG, fsImage); } } } @@ -4503,7 +4505,7 @@ void registerDatanode(DatanodeRegistration nodeReg) throws IOException { * @return registration ID */ String getRegistrationID() { - return Storage.getRegistrationID(dir.fsImage.getStorage()); + return Storage.getRegistrationID(getFSImage().getStorage()); } /** @@ -4719,7 +4721,7 @@ public void stop() { } public FSImage getFSImage() { - return dir.fsImage; + return fsImage; } public FSEditLog getEditLog() { @@ -7044,7 +7046,7 @@ private long getDfsUsed(DatanodeDescriptor alivenode) { @Override // NameNodeMXBean public String getClusterId() { - return dir.fsImage.getStorage().getClusterID(); + return getFSImage().getStorage().getClusterID(); } @Override // NameNodeMXBean diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index 9747f9cddac..110e020d18f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -831,7 +831,7 @@ public boolean isInSafeMode() { /** get FSImage */ @VisibleForTesting public FSImage getFSImage() { - return namesystem.dir.fsImage; + return namesystem.getFSImage(); } /** @@ -1141,7 +1141,7 @@ public static boolean doRollback(Configuration conf, return true; } } - nsys.dir.fsImage.doRollback(nsys); + nsys.getFSImage().doRollback(nsys); return false; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java index f26d3ccf9c5..c32ed67d6e4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java @@ -45,6 +45,7 @@ import org.apache.hadoop.ipc.StandbyException; import org.apache.hadoop.security.AccessControlException; import org.mockito.Mockito; +import org.mockito.internal.util.reflection.Whitebox; /** * This is a utility class to expose NameNode functionality for unit tests. @@ -177,8 +178,9 @@ public static ReentrantReadWriteLock spyOnFsLock(FSNamesystem fsn) { } public static FSImage spyOnFsImage(NameNode nn1) { - FSImage spy = Mockito.spy(nn1.getNamesystem().dir.fsImage); - nn1.getNamesystem().dir.fsImage = spy; + FSNamesystem fsn = nn1.getNamesystem(); + FSImage spy = Mockito.spy(fsn.getFSImage()); + Whitebox.setInternalState(fsn, "fsImage", spy); return spy; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSPermissionChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSPermissionChecker.java index 2868133cdaf..b1c5ca7c557 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSPermissionChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSPermissionChecker.java @@ -75,8 +75,7 @@ public Object answer(InvocationOnMock invocation) throws Throwable { return new PermissionStatus(SUPERUSER, SUPERGROUP, perm); } }).when(fsn).createFsOwnerPermissions(any(FsPermission.class)); - FSImage image = mock(FSImage.class); - dir = new FSDirectory(image, fsn, conf); + dir = new FSDirectory(fsn, conf); inodeRoot = dir.getRoot(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java index 5231fc3b611..4ef87984cf9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSaveNamespace.java @@ -61,6 +61,7 @@ import org.apache.log4j.Level; import org.junit.Test; import org.mockito.Mockito; +import org.mockito.internal.util.reflection.Whitebox; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; @@ -124,14 +125,14 @@ private void saveNamespaceWithInjectedFault(Fault fault) throws Exception { FSNamesystem fsn = FSNamesystem.loadFromDisk(conf); // Replace the FSImage with a spy - FSImage originalImage = fsn.dir.fsImage; + FSImage originalImage = fsn.getFSImage(); NNStorage storage = originalImage.getStorage(); NNStorage spyStorage = spy(storage); originalImage.storage = spyStorage; FSImage spyImage = spy(originalImage); - fsn.dir.fsImage = spyImage; + Whitebox.setInternalState(fsn, "fsImage", spyImage); boolean shouldFail = false; // should we expect the save operation to fail // inject fault @@ -233,11 +234,11 @@ public void testReinsertnamedirsInSavenamespace() throws Exception { FSNamesystem fsn = FSNamesystem.loadFromDisk(conf); // Replace the FSImage with a spy - FSImage originalImage = fsn.dir.fsImage; + FSImage originalImage = fsn.getFSImage(); NNStorage storage = originalImage.getStorage(); FSImage spyImage = spy(originalImage); - fsn.dir.fsImage = spyImage; + Whitebox.setInternalState(fsn, "fsImage", spyImage); FileSystem fs = FileSystem.getLocal(conf); File rootDir = storage.getStorageDir(0).getRoot(); @@ -367,14 +368,15 @@ public void doTestFailedSaveNamespace(boolean restoreStorageAfterFailure) FSNamesystem fsn = FSNamesystem.loadFromDisk(conf); // Replace the FSImage with a spy - final FSImage originalImage = fsn.dir.fsImage; + final FSImage originalImage = fsn.getFSImage(); NNStorage storage = originalImage.getStorage(); storage.close(); // unlock any directories that FSNamesystem's initialization may have locked NNStorage spyStorage = spy(storage); originalImage.storage = spyStorage; FSImage spyImage = spy(originalImage); - fsn.dir.fsImage = spyImage; + Whitebox.setInternalState(fsn, "fsImage", spyImage); + spyImage.storage.setStorageDirectories( FSNamesystem.getNamespaceDirs(conf), FSNamesystem.getNamespaceEditsDirs(conf)); @@ -504,7 +506,7 @@ public void testCancelSaveNamespace() throws Exception { FSNamesystem fsn = FSNamesystem.loadFromDisk(conf); // Replace the FSImage with a spy - final FSImage image = fsn.dir.fsImage; + final FSImage image = fsn.getFSImage(); NNStorage storage = image.getStorage(); storage.close(); // unlock any directories that FSNamesystem's initialization may have locked storage.setStorageDirectories( From bfc9c4b5a567a9f2efc185a7f25e17edd536101b Mon Sep 17 00:00:00 2001 From: Haohui Mai Date: Fri, 20 Jun 2014 22:23:31 +0000 Subject: [PATCH 013/112] HADOOP-10479. Fix new findbugs warnings in hadoop-minikdc. Contributed by Swarnim Kulkarni. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1604292 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../src/main/java/org/apache/hadoop/minikdc/MiniKdc.java | 9 +++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 3adee445e54..cb49bc5343f 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -617,6 +617,9 @@ Release 2.5.0 - UNRELEASED HADOOP-10711. Cleanup some extra dependencies from hadoop-auth. (rkanter via tucu) + HADOOP-10479. Fix new findbugs warnings in hadoop-minikdc. + (Swarnim Kulkarni via wheat9) + Release 2.4.1 - 2014-06-23 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-minikdc/src/main/java/org/apache/hadoop/minikdc/MiniKdc.java b/hadoop-common-project/hadoop-minikdc/src/main/java/org/apache/hadoop/minikdc/MiniKdc.java index 33ff1bb9c22..d3ea2e70cfa 100644 --- a/hadoop-common-project/hadoop-minikdc/src/main/java/org/apache/hadoop/minikdc/MiniKdc.java +++ b/hadoop-common-project/hadoop-minikdc/src/main/java/org/apache/hadoop/minikdc/MiniKdc.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.minikdc; +import org.apache.commons.io.Charsets; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.text.StrSubstitutor; @@ -56,7 +57,7 @@ import java.io.BufferedReader; import java.io.File; -import java.io.FileReader; +import java.io.FileInputStream; import java.io.InputStream; import java.io.InputStreamReader; import java.io.StringReader; @@ -126,9 +127,9 @@ public static void main(String[] args) throws Exception { + file.getAbsolutePath()); } Properties userConf = new Properties(); - FileReader r = null; + InputStreamReader r = null; try { - r = new FileReader(file); + r = new InputStreamReader(new FileInputStream(file), Charsets.UTF_8); userConf.load(r); } finally { if (r != null) { @@ -438,7 +439,7 @@ private void initKDCServer() throws Exception { BufferedReader r = null; try { - r = new BufferedReader(new InputStreamReader(is2)); + r = new BufferedReader(new InputStreamReader(is2, Charsets.UTF_8)); String line = r.readLine(); while (line != null) { From 0c5128969522cf754010c32cdcbfcfa5ebe5b3b0 Mon Sep 17 00:00:00 2001 From: Chris Nauroth Date: Fri, 20 Jun 2014 23:58:23 +0000 Subject: [PATCH 014/112] HDFS-6222. Remove background token renewer from webhdfs. Contributed by Rushabh Shah and Daryn Sharp. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1604300 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../delegation/DelegationTokenIdentifier.java | 23 ++ .../hadoop/hdfs/web/SWebHdfsFileSystem.java | 4 +- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 140 ++++++-- ...ache.hadoop.security.token.TokenIdentifier | 2 + .../hadoop/hdfs/web/TestWebHdfsTokens.java | 309 +++++++++++++++++- 6 files changed, 439 insertions(+), 42 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 44020ead059..5cbea0dd7cf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -684,6 +684,9 @@ Release 2.5.0 - UNRELEASED HDFS-6535. HDFS quota update is wrong when file is appended. (George Wong via jing9) + HDFS-6222. Remove background token renewer from webhdfs. + (Rushabh Shah and Daryn Sharp via cnauroth) + BREAKDOWN OF HDFS-2006 SUBTASKS AND RELATED JIRAS HDFS-6299. Protobuf for XAttr and client-side implementation. (Yi Liu via umamahesh) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java index a5e14cba945..07052ddf760 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenIdentifier.java @@ -23,6 +23,8 @@ import java.io.IOException; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hdfs.web.SWebHdfsFileSystem; +import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; @@ -75,4 +77,25 @@ public static String stringifyToken(final Token token) throws IOException { return ident.toString(); } } + + public static class WebHdfsDelegationTokenIdentifier + extends DelegationTokenIdentifier { + public WebHdfsDelegationTokenIdentifier() { + super(); + } + @Override + public Text getKind() { + return WebHdfsFileSystem.TOKEN_KIND; + } + } + + public static class SWebHdfsDelegationTokenIdentifier extends WebHdfsDelegationTokenIdentifier { + public SWebHdfsDelegationTokenIdentifier() { + super(); + } + @Override + public Text getKind() { + return SWebHdfsFileSystem.TOKEN_KIND; + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/SWebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/SWebHdfsFileSystem.java index 3c65ad83e28..fa89ec3551b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/SWebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/SWebHdfsFileSystem.java @@ -36,8 +36,8 @@ protected String getTransportScheme() { } @Override - protected synchronized void initializeTokenAspect() { - tokenAspect = new TokenAspect(this, tokenServiceName, TOKEN_KIND); + protected Text getTokenKind() { + return TOKEN_KIND; } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index f8dcc398bd2..94c666a3a11 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -69,11 +69,14 @@ import org.apache.hadoop.io.retry.RetryUtils; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.authentication.client.AuthenticationException; +import org.apache.hadoop.security.token.SecretManager.InvalidToken; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.security.token.TokenSelector; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSelector; import org.apache.hadoop.util.Progressable; import org.mortbay.util.ajax.JSON; @@ -98,7 +101,7 @@ public class WebHdfsFileSystem extends FileSystem /** Delegation token kind */ public static final Text TOKEN_KIND = new Text("WEBHDFS delegation"); - protected TokenAspect tokenAspect; + private boolean canRefreshDelegationToken; private UserGroupInformation ugi; private URI uri; @@ -127,13 +130,8 @@ protected String getTransportScheme() { return "http"; } - /** - * Initialize tokenAspect. This function is intended to - * be overridden by SWebHdfsFileSystem. - */ - protected synchronized void initializeTokenAspect() { - tokenAspect = new TokenAspect(this, tokenServiceName, - TOKEN_KIND); + protected Text getTokenKind() { + return TOKEN_KIND; } @Override @@ -162,7 +160,6 @@ public synchronized void initialize(URI uri, Configuration conf this.tokenServiceName = isLogicalUri ? HAUtil.buildTokenServiceForLogicalUri(uri) : SecurityUtil.buildTokenService(getCanonicalUri()); - initializeTokenAspect(); if (!isHA) { this.retryPolicy = @@ -195,10 +192,8 @@ public synchronized void initialize(URI uri, Configuration conf } this.workingDir = getHomeDirectory(); - - if (UserGroupInformation.isSecurityEnabled()) { - tokenAspect.initDelegationToken(ugi); - } + this.canRefreshDelegationToken = UserGroupInformation.isSecurityEnabled(); + this.delegationToken = null; } @Override @@ -213,11 +208,46 @@ public static boolean isEnabled(final Configuration conf, final Log log) { return b; } + TokenSelector tokenSelector = + new AbstractDelegationTokenSelector(getTokenKind()){}; + + // the first getAuthParams() for a non-token op will either get the + // internal token from the ugi or lazy fetch one protected synchronized Token getDelegationToken() throws IOException { - tokenAspect.ensureTokenInitialized(); + if (canRefreshDelegationToken && delegationToken == null) { + Token token = tokenSelector.selectToken( + new Text(getCanonicalServiceName()), ugi.getTokens()); + // ugi tokens are usually indicative of a task which can't + // refetch tokens. even if ugi has credentials, don't attempt + // to get another token to match hdfs/rpc behavior + if (token != null) { + LOG.debug("Using UGI token: " + token); + canRefreshDelegationToken = false; + } else { + token = getDelegationToken(null); + if (token != null) { + LOG.debug("Fetched new token: " + token); + } else { // security is disabled + canRefreshDelegationToken = false; + } + } + setDelegationToken(token); + } return delegationToken; } + @VisibleForTesting + synchronized boolean replaceExpiredDelegationToken() throws IOException { + boolean replaced = false; + if (canRefreshDelegationToken) { + Token token = getDelegationToken(null); + LOG.debug("Replaced expired token: " + token); + setDelegationToken(token); + replaced = (token != null); + } + return replaced; + } + @Override protected int getDefaultPort() { return DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT; @@ -288,8 +318,8 @@ private Path makeAbsolute(Path f) { final int code = conn.getResponseCode(); // server is demanding an authentication we don't support if (code == HttpURLConnection.HTTP_UNAUTHORIZED) { - throw new IOException( - new AuthenticationException(conn.getResponseMessage())); + // match hdfs/rpc exception + throw new AccessControlException(conn.getResponseMessage()); } if (code != op.getExpectedHttpResponseCode()) { final Map m; @@ -309,7 +339,15 @@ private Path makeAbsolute(Path f) { return m; } - final RemoteException re = JsonUtil.toRemoteException(m); + IOException re = JsonUtil.toRemoteException(m); + // extract UGI-related exceptions and unwrap InvalidToken + // the NN mangles these exceptions but the DN does not and may need + // to re-fetch a token if either report the token is expired + if (re.getMessage().startsWith("Failed to obtain user group information:")) { + String[] parts = re.getMessage().split(":\\s+", 3); + re = new RemoteException(parts[1], parts[2]); + re = ((RemoteException)re).unwrapRemoteException(InvalidToken.class); + } throw unwrapException? toIOException(re): re; } return null; @@ -369,7 +407,7 @@ Param[] getAuthParameters(final HttpOpParam.Op op) throws IOException { // Skip adding delegation token for token operations because these // operations require authentication. Token token = null; - if (UserGroupInformation.isSecurityEnabled() && !op.getRequireAuth()) { + if (!op.getRequireAuth()) { token = getDelegationToken(); } if (token != null) { @@ -540,11 +578,17 @@ private T runWithRetry() throws IOException { validateResponse(op, conn, false); } return getResponse(conn); - } catch (IOException ioe) { - Throwable cause = ioe.getCause(); - if (cause != null && cause instanceof AuthenticationException) { - throw ioe; // no retries for auth failures + } catch (AccessControlException ace) { + // no retries for auth failures + throw ace; + } catch (InvalidToken it) { + // try to replace the expired token with a new one. the attempt + // to acquire a new token must be outside this operation's retry + // so if it fails after its own retries, this operation fails too. + if (op.getRequireAuth() || !replaceExpiredDelegationToken()) { + throw it; } + } catch (IOException ioe) { shouldRetry(ioe, retry); } } @@ -712,6 +756,17 @@ public void close() throws IOException { }; } } + + class FsPathConnectionRunner extends AbstractFsPathRunner { + FsPathConnectionRunner(Op op, Path fspath, Param... parameters) { + super(op, fspath, parameters); + } + @Override + HttpURLConnection getResponse(final HttpURLConnection conn) + throws IOException { + return conn; + } + } /** * Used by open() which tracks the resolved url itself @@ -1077,16 +1132,41 @@ public FSDataInputStream open(final Path f, final int buffersize ) throws IOException { statistics.incrementReadOps(1); final HttpOpParam.Op op = GetOpParam.Op.OPEN; - final URL url = toUrl(op, f, new BufferSizeParam(buffersize)); + // use a runner so the open can recover from an invalid token + FsPathConnectionRunner runner = + new FsPathConnectionRunner(op, f, new BufferSizeParam(buffersize)); return new FSDataInputStream(new OffsetUrlInputStream( - new OffsetUrlOpener(url), new OffsetUrlOpener(null))); + new UnresolvedUrlOpener(runner), new OffsetUrlOpener(null))); } @Override - public void close() throws IOException { - super.close(); - synchronized (this) { - tokenAspect.removeRenewAction(); + public synchronized void close() throws IOException { + try { + if (canRefreshDelegationToken && delegationToken != null) { + cancelDelegationToken(delegationToken); + } + } catch (IOException ioe) { + LOG.debug("Token cancel failed: "+ioe); + } finally { + super.close(); + } + } + + // use FsPathConnectionRunner to ensure retries for InvalidTokens + class UnresolvedUrlOpener extends ByteRangeInputStream.URLOpener { + private final FsPathConnectionRunner runner; + UnresolvedUrlOpener(FsPathConnectionRunner runner) { + super(null); + this.runner = runner; + } + + @Override + protected HttpURLConnection connect(long offset, boolean resolved) + throws IOException { + assert offset == 0; + HttpURLConnection conn = runner.run(); + setURL(conn.getURL()); + return conn; } } @@ -1139,7 +1219,7 @@ static URL removeOffsetParam(final URL url) throws MalformedURLException { } static class OffsetUrlInputStream extends ByteRangeInputStream { - OffsetUrlInputStream(OffsetUrlOpener o, OffsetUrlOpener r) + OffsetUrlInputStream(UnresolvedUrlOpener o, OffsetUrlOpener r) throws IOException { super(o, r); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier index 59603a96eb1..b6b6171f944 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier @@ -13,3 +13,5 @@ # org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier +org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier$WebHdfsDelegationTokenIdentifier +org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier$SWebHdfsDelegationTokenIdentifier diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java index 2b3685e9d81..44e4961bb9d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java @@ -19,47 +19,63 @@ package org.apache.hadoop.hdfs.web; import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS; +import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.SIMPLE; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.reset; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.verify; +import static org.mockito.Matchers.*; +import static org.mockito.Mockito.*; +import java.io.File; import java.io.IOException; +import java.io.InputStream; +import java.net.InetSocketAddress; import java.net.URI; +import java.security.PrivilegedExceptionAction; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSUtil; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.web.resources.DeleteOpParam; import org.apache.hadoop.hdfs.web.resources.GetOpParam; import org.apache.hadoop.hdfs.web.resources.HttpOpParam; import org.apache.hadoop.hdfs.web.resources.PostOpParam; import org.apache.hadoop.hdfs.web.resources.PutOpParam; +import org.apache.hadoop.http.HttpConfig; +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.ssl.KeyStoreTestUtil; +import org.apache.hadoop.security.token.SecretManager.InvalidToken; import org.apache.hadoop.security.token.Token; +import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; -import org.mockito.internal.util.reflection.Whitebox; public class TestWebHdfsTokens { private static Configuration conf; + URI uri = null; @BeforeClass public static void setUp() { conf = new Configuration(); SecurityUtil.setAuthenticationMethod(KERBEROS, conf); UserGroupInformation.setConfiguration(conf); + UserGroupInformation.setLoginUser( + UserGroupInformation.createUserForTesting( + "LoginUser", new String[]{"supergroup"})); } private WebHdfsFileSystem spyWebhdfsInSecureSetup() throws IOException { WebHdfsFileSystem fsOrig = new WebHdfsFileSystem(); fsOrig.initialize(URI.create("webhdfs://127.0.0.1:0"), conf); WebHdfsFileSystem fs = spy(fsOrig); - Whitebox.setInternalState(fsOrig.tokenAspect, "fs", fs); return fs; } @@ -89,7 +105,7 @@ public void testNoTokenForGetToken() throws IOException { } @Test(timeout = 5000) - public void testNoTokenForCanclToken() throws IOException { + public void testNoTokenForRenewToken() throws IOException { checkNoTokenForOperation(PutOpParam.Op.RENEWDELEGATIONTOKEN); } @@ -139,4 +155,277 @@ public void testDeleteOpRequireAuth() { assertFalse(op.getRequireAuth()); } } + + @SuppressWarnings("unchecked") // for any(Token.class) + @Test + public void testLazyTokenFetchForWebhdfs() throws Exception { + MiniDFSCluster cluster = null; + WebHdfsFileSystem fs = null; + try { + final Configuration clusterConf = new HdfsConfiguration(conf); + SecurityUtil.setAuthenticationMethod(SIMPLE, clusterConf); + clusterConf.setBoolean(DFSConfigKeys + .DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true); + + // trick the NN into thinking security is enabled w/o it trying + // to login from a keytab + UserGroupInformation.setConfiguration(clusterConf); + cluster = new MiniDFSCluster.Builder(clusterConf).numDataNodes(1).build(); + cluster.waitActive(); + SecurityUtil.setAuthenticationMethod(KERBEROS, clusterConf); + UserGroupInformation.setConfiguration(clusterConf); + + uri = DFSUtil.createUri( + "webhdfs", cluster.getNameNode().getHttpAddress()); + validateLazyTokenFetch(clusterConf); + } finally { + IOUtils.cleanup(null, fs); + if (cluster != null) { + cluster.shutdown(); + } + } + } + + @SuppressWarnings("unchecked") // for any(Token.class) + @Test + public void testLazyTokenFetchForSWebhdfs() throws Exception { + MiniDFSCluster cluster = null; + SWebHdfsFileSystem fs = null; + try { + final Configuration clusterConf = new HdfsConfiguration(conf); + SecurityUtil.setAuthenticationMethod(SIMPLE, clusterConf); + clusterConf.setBoolean(DFSConfigKeys + .DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true); + String BASEDIR = System.getProperty("test.build.dir", + "target/test-dir") + "/" + TestWebHdfsTokens.class.getSimpleName(); + String keystoresDir; + String sslConfDir; + + clusterConf.setBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY, true); + clusterConf.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY, HttpConfig.Policy.HTTPS_ONLY.name()); + clusterConf.set(DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY, "localhost:0"); + clusterConf.set(DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY, "localhost:0"); + + File base = new File(BASEDIR); + FileUtil.fullyDelete(base); + base.mkdirs(); + keystoresDir = new File(BASEDIR).getAbsolutePath(); + sslConfDir = KeyStoreTestUtil.getClasspathDir(TestWebHdfsTokens.class); + KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, clusterConf, false); + + // trick the NN into thinking security is enabled w/o it trying + // to login from a keytab + UserGroupInformation.setConfiguration(clusterConf); + cluster = new MiniDFSCluster.Builder(clusterConf).numDataNodes(1).build(); + cluster.waitActive(); + InetSocketAddress addr = cluster.getNameNode().getHttpsAddress(); + String nnAddr = NetUtils.getHostPortString(addr); + clusterConf.set(DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY, nnAddr); + SecurityUtil.setAuthenticationMethod(KERBEROS, clusterConf); + UserGroupInformation.setConfiguration(clusterConf); + + uri = DFSUtil.createUri( + "swebhdfs", cluster.getNameNode().getHttpsAddress()); + validateLazyTokenFetch(clusterConf); + } finally { + IOUtils.cleanup(null, fs); + if (cluster != null) { + cluster.shutdown(); + } + } + } + + @SuppressWarnings("unchecked") + private void validateLazyTokenFetch(final Configuration clusterConf) throws Exception{ + final String testUser = "DummyUser"; + UserGroupInformation ugi = UserGroupInformation.createUserForTesting( + testUser, new String[]{"supergroup"}); + WebHdfsFileSystem fs = ugi.doAs(new PrivilegedExceptionAction() { + @Override + public WebHdfsFileSystem run() throws IOException { + return spy((WebHdfsFileSystem) FileSystem.newInstance(uri, clusterConf)); + } + }); + // verify token ops don't get a token + Assert.assertNull(fs.getRenewToken()); + Token token = fs.getDelegationToken(null); + fs.renewDelegationToken(token); + fs.cancelDelegationToken(token); + verify(fs, never()).getDelegationToken(); + verify(fs, never()).replaceExpiredDelegationToken(); + verify(fs, never()).setDelegationToken(any(Token.class)); + Assert.assertNull(fs.getRenewToken()); + reset(fs); + + // verify first non-token op gets a token + final Path p = new Path("/f"); + fs.create(p, (short)1).close(); + verify(fs, times(1)).getDelegationToken(); + verify(fs, never()).replaceExpiredDelegationToken(); + verify(fs, times(1)).getDelegationToken(anyString()); + verify(fs, times(1)).setDelegationToken(any(Token.class)); + token = fs.getRenewToken(); + Assert.assertNotNull(token); + Assert.assertEquals(testUser, getTokenOwner(token)); + Assert.assertEquals(fs.getTokenKind(), token.getKind()); + reset(fs); + + // verify prior token is reused + fs.getFileStatus(p); + verify(fs, times(1)).getDelegationToken(); + verify(fs, never()).replaceExpiredDelegationToken(); + verify(fs, never()).getDelegationToken(anyString()); + verify(fs, never()).setDelegationToken(any(Token.class)); + Token token2 = fs.getRenewToken(); + Assert.assertNotNull(token2); + Assert.assertEquals(fs.getTokenKind(), token.getKind()); + Assert.assertSame(token, token2); + reset(fs); + + // verify renew of expired token fails w/o getting a new token + token = fs.getRenewToken(); + fs.cancelDelegationToken(token); + try { + fs.renewDelegationToken(token); + Assert.fail("should have failed"); + } catch (InvalidToken it) { + } catch (Exception ex) { + Assert.fail("wrong exception:"+ex); + } + verify(fs, never()).getDelegationToken(); + verify(fs, never()).replaceExpiredDelegationToken(); + verify(fs, never()).getDelegationToken(anyString()); + verify(fs, never()).setDelegationToken(any(Token.class)); + token2 = fs.getRenewToken(); + Assert.assertNotNull(token2); + Assert.assertEquals(fs.getTokenKind(), token.getKind()); + Assert.assertSame(token, token2); + reset(fs); + + // verify cancel of expired token fails w/o getting a new token + try { + fs.cancelDelegationToken(token); + Assert.fail("should have failed"); + } catch (InvalidToken it) { + } catch (Exception ex) { + Assert.fail("wrong exception:"+ex); + } + verify(fs, never()).getDelegationToken(); + verify(fs, never()).replaceExpiredDelegationToken(); + verify(fs, never()).getDelegationToken(anyString()); + verify(fs, never()).setDelegationToken(any(Token.class)); + token2 = fs.getRenewToken(); + Assert.assertNotNull(token2); + Assert.assertEquals(fs.getTokenKind(), token.getKind()); + Assert.assertSame(token, token2); + reset(fs); + + // verify an expired token is replaced with a new token + fs.open(p).close(); + verify(fs, times(2)).getDelegationToken(); // first bad, then good + verify(fs, times(1)).replaceExpiredDelegationToken(); + verify(fs, times(1)).getDelegationToken(null); + verify(fs, times(1)).setDelegationToken(any(Token.class)); + token2 = fs.getRenewToken(); + Assert.assertNotNull(token2); + Assert.assertNotSame(token, token2); + Assert.assertEquals(fs.getTokenKind(), token.getKind()); + Assert.assertEquals(testUser, getTokenOwner(token2)); + reset(fs); + + // verify with open because it's a little different in how it + // opens connections + fs.cancelDelegationToken(fs.getRenewToken()); + InputStream is = fs.open(p); + is.read(); + is.close(); + verify(fs, times(2)).getDelegationToken(); // first bad, then good + verify(fs, times(1)).replaceExpiredDelegationToken(); + verify(fs, times(1)).getDelegationToken(null); + verify(fs, times(1)).setDelegationToken(any(Token.class)); + token2 = fs.getRenewToken(); + Assert.assertNotNull(token2); + Assert.assertNotSame(token, token2); + Assert.assertEquals(fs.getTokenKind(), token.getKind()); + Assert.assertEquals(testUser, getTokenOwner(token2)); + reset(fs); + + // verify fs close cancels the token + fs.close(); + verify(fs, never()).getDelegationToken(); + verify(fs, never()).replaceExpiredDelegationToken(); + verify(fs, never()).getDelegationToken(anyString()); + verify(fs, never()).setDelegationToken(any(Token.class)); + verify(fs, times(1)).cancelDelegationToken(eq(token2)); + + // add a token to ugi for a new fs, verify it uses that token + token = fs.getDelegationToken(null); + ugi.addToken(token); + fs = ugi.doAs(new PrivilegedExceptionAction() { + @Override + public WebHdfsFileSystem run() throws IOException { + return spy((WebHdfsFileSystem) FileSystem.newInstance(uri, clusterConf)); + } + }); + Assert.assertNull(fs.getRenewToken()); + fs.getFileStatus(new Path("/")); + verify(fs, times(1)).getDelegationToken(); + verify(fs, never()).replaceExpiredDelegationToken(); + verify(fs, never()).getDelegationToken(anyString()); + verify(fs, times(1)).setDelegationToken(eq(token)); + token2 = fs.getRenewToken(); + Assert.assertNotNull(token2); + Assert.assertEquals(fs.getTokenKind(), token.getKind()); + Assert.assertSame(token, token2); + reset(fs); + + // verify it reuses the prior ugi token + fs.getFileStatus(new Path("/")); + verify(fs, times(1)).getDelegationToken(); + verify(fs, never()).replaceExpiredDelegationToken(); + verify(fs, never()).getDelegationToken(anyString()); + verify(fs, never()).setDelegationToken(any(Token.class)); + token2 = fs.getRenewToken(); + Assert.assertNotNull(token2); + Assert.assertEquals(fs.getTokenKind(), token.getKind()); + Assert.assertSame(token, token2); + reset(fs); + + // verify an expired ugi token is NOT replaced with a new token + fs.cancelDelegationToken(token); + for (int i=0; i<2; i++) { + try { + fs.getFileStatus(new Path("/")); + Assert.fail("didn't fail"); + } catch (InvalidToken it) { + } catch (Exception ex) { + Assert.fail("wrong exception:"+ex); + } + verify(fs, times(1)).getDelegationToken(); + verify(fs, times(1)).replaceExpiredDelegationToken(); + verify(fs, never()).getDelegationToken(anyString()); + verify(fs, never()).setDelegationToken(any(Token.class)); + token2 = fs.getRenewToken(); + Assert.assertNotNull(token2); + Assert.assertEquals(fs.getTokenKind(), token.getKind()); + Assert.assertSame(token, token2); + reset(fs); + } + + // verify fs close does NOT cancel the ugi token + fs.close(); + verify(fs, never()).getDelegationToken(); + verify(fs, never()).replaceExpiredDelegationToken(); + verify(fs, never()).getDelegationToken(anyString()); + verify(fs, never()).setDelegationToken(any(Token.class)); + verify(fs, never()).cancelDelegationToken(any(Token.class)); + } + + private String getTokenOwner(Token token) throws IOException { + // webhdfs doesn't register properly with the class loader + @SuppressWarnings({ "rawtypes", "unchecked" }) + Token clone = new Token(token); + clone.setKind(DelegationTokenIdentifier.HDFS_DELEGATION_KIND); + return clone.decodeIdentifier().getUser().getUserName(); + } } From 905c58ed277079262d7a79cde638e41d50f30941 Mon Sep 17 00:00:00 2001 From: Arun Murthy Date: Sat, 21 Jun 2014 05:33:35 +0000 Subject: [PATCH 015/112] Updated release notes for hadoop-2.4.1 rc1. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1604314 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/src/main/docs/releasenotes.html | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html b/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html index 6a6d4b4556a..b3a4a13a079 100644 --- a/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html +++ b/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html @@ -470,6 +470,10 @@

Changes since Hadoop 2.4.0

Major bug reported by Jian He and fixed by Vinod Kumar Vavilapalli
Few tests in TestJobClient fail on Windows
+
  • MAPREDUCE-5830. + Blocker bug reported by Jason Lowe and fixed by Akira AJISAKA
    + HostUtil.getTaskLogUrl is not backwards binary compatible with 2.3
    +
  • MAPREDUCE-5828. Major bug reported by Vinod Kumar Vavilapalli and fixed by Vinod Kumar Vavilapalli
    TestMapReduceJobControl fails on JDK 7 + Windows
    @@ -506,6 +510,10 @@

    Changes since Hadoop 2.4.0

    Trivial bug reported by Todd Lipcon and fixed by Chen He
    docs for map output compression incorrectly reference SequenceFile
    • +
  • HDFS-6527. + Blocker bug reported by Kihwal Lee and fixed by Kihwal Lee
    + Edit log corruption due to defered INode removal
    +
  • HDFS-6411. Major bug reported by Zhongyi Xie and fixed by Brandon Li (nfs)
    nfs-hdfs-gateway mount raises I/O error and hangs when a unauthorized user attempts to access it
    From 6fcbf9b848c63465d26a40387a9be212e708f80b Mon Sep 17 00:00:00 2001 From: Karthik Kambatla Date: Sat, 21 Jun 2014 07:30:07 +0000 Subject: [PATCH 016/112] YARN-2187. FairScheduler: Disable max-AM-share check by default. (Robert Kanter via kasha) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1604321 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../fair/AllocationConfiguration.java | 2 +- .../fair/AllocationFileLoaderService.java | 2 +- .../scheduler/fair/FSLeafQueue.java | 3 + .../scheduler/fair/TestFairScheduler.java | 86 +++++++++++++++++++ .../src/site/apt/FairScheduler.apt.vm | 5 +- 6 files changed, 97 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 77929357871..52a61809fb9 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -259,6 +259,9 @@ Release 2.5.0 - UNRELEASED NMLeveldbStateStoreService#loadLocalizationState() within finally block (Junping Du via jlowe) + YARN-2187. FairScheduler: Disable max-AM-share check by default. + (Robert Kanter via kasha) + Release 2.4.1 - 2014-06-23 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java index 237cad29c19..d4ba88faf14 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java @@ -126,7 +126,7 @@ public AllocationConfiguration(Configuration conf) { queueMaxAMShares = new HashMap(); userMaxAppsDefault = Integer.MAX_VALUE; queueMaxAppsDefault = Integer.MAX_VALUE; - queueMaxAMShareDefault = 1.0f; + queueMaxAMShareDefault = -1.0f; queueAcls = new HashMap>(); minSharePreemptionTimeouts = new HashMap(); defaultMinSharePreemptionTimeout = Long.MAX_VALUE; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java index 064bdfc817f..4cc88c140d4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java @@ -221,7 +221,7 @@ public synchronized void reloadAllocations() throws IOException, new HashMap>(); int userMaxAppsDefault = Integer.MAX_VALUE; int queueMaxAppsDefault = Integer.MAX_VALUE; - float queueMaxAMShareDefault = 1.0f; + float queueMaxAMShareDefault = -1.0f; long fairSharePreemptionTimeout = Long.MAX_VALUE; long defaultMinSharePreemptionTimeout = Long.MAX_VALUE; SchedulingPolicy defaultSchedPolicy = SchedulingPolicy.DEFAULT_POLICY; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java index 21dbdc5faca..8f957382e6a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java @@ -308,6 +308,9 @@ public ActiveUsersManager getActiveUsersManager() { public boolean canRunAppAM(Resource amResource) { float maxAMShare = scheduler.getAllocationConfiguration().getQueueMaxAMShare(getName()); + if (Math.abs(maxAMShare - -1.0f) < 0.0001) { + return true; + } Resource maxAMResource = Resources.multiply(getFairShare(), maxAMShare); Resource ifRunAMResource = Resources.add(amResourceUsage, amResource); return !policy diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 9d8b1d117d0..dd42aa0e183 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -2483,6 +2483,92 @@ public void testQueueMaxAMShare() throws Exception { 0, queue1.getAmResourceUsage().getMemory()); } + @Test + public void testQueueMaxAMShareDefault() throws Exception { + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println("1.0"); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.close(); + + scheduler.init(conf); + scheduler.start(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + RMNode node = + MockNodes.newNodeInfo(1, Resources.createResource(8192, 20), + 0, "127.0.0.1"); + NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node); + NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node); + scheduler.handle(nodeEvent); + scheduler.update(); + + FSLeafQueue queue1 = + scheduler.getQueueManager().getLeafQueue("queue1", true); + assertEquals("Queue queue1's fair share should be 1366", + 1366, queue1.getFairShare().getMemory()); + FSLeafQueue queue2 = + scheduler.getQueueManager().getLeafQueue("queue2", true); + assertEquals("Queue queue2's fair share should be 1366", + 1366, queue2.getFairShare().getMemory()); + FSLeafQueue queue3 = + scheduler.getQueueManager().getLeafQueue("queue3", true); + assertEquals("Queue queue3's fair share should be 1366", + 1366, queue3.getFairShare().getMemory()); + FSLeafQueue queue4 = + scheduler.getQueueManager().getLeafQueue("queue4", true); + assertEquals("Queue queue4's fair share should be 1366", + 1366, queue4.getFairShare().getMemory()); + FSLeafQueue queue5 = + scheduler.getQueueManager().getLeafQueue("queue5", true); + assertEquals("Queue queue5's fair share should be 1366", + 1366, queue5.getFairShare().getMemory()); + + Resource amResource1 = Resource.newInstance(2048, 1); + int amPriority = RMAppAttemptImpl.AM_CONTAINER_PRIORITY.getPriority(); + + // Exceeds queue limit, but default maxAMShare is -1.0 so it doesn't matter + ApplicationAttemptId attId1 = createAppAttemptId(1, 1); + createApplicationWithAMResource(attId1, "queue1", "test1", amResource1); + createSchedulingRequestExistingApplication(2048, 1, amPriority, attId1); + FSSchedulerApp app1 = scheduler.getSchedulerApp(attId1); + scheduler.update(); + scheduler.handle(updateEvent); + assertEquals("Application1's AM requests 2048 MB memory", + 2048, app1.getAMResource().getMemory()); + assertEquals("Application1's AM should be running", + 1, app1.getLiveContainers().size()); + assertEquals("Queue1's AM resource usage should be 2048 MB memory", + 2048, queue1.getAmResourceUsage().getMemory()); + + // Exceeds queue limit, and maxAMShare is 1.0 + ApplicationAttemptId attId2 = createAppAttemptId(2, 1); + createApplicationWithAMResource(attId2, "queue2", "test1", amResource1); + createSchedulingRequestExistingApplication(2048, 1, amPriority, attId2); + FSSchedulerApp app2 = scheduler.getSchedulerApp(attId2); + scheduler.update(); + scheduler.handle(updateEvent); + assertEquals("Application2's AM requests 2048 MB memory", + 2048, app2.getAMResource().getMemory()); + assertEquals("Application2's AM should not be running", + 0, app2.getLiveContainers().size()); + assertEquals("Queue2's AM resource usage should be 0 MB memory", + 0, queue2.getAmResourceUsage().getMemory()); + } + @Test public void testMaxRunningAppsHierarchicalQueues() throws Exception { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm index 23faf27bf26..b9cda2c254d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm @@ -239,8 +239,9 @@ Allocation file format * maxAMShare: limit the fraction of the queue's fair share that can be used to run application masters. This property can only be used for leaf queues. - Default value is 1.0f, which means AMs in the leaf queue can take up to 100% - of both the memory and CPU fair share. + For example, if set to 1.0f, then AMs in the leaf queue can take up to 100% + of both the memory and CPU fair share. The default value is -1.0f, which + means that this check is disabled. * weight: to share the cluster non-proportionally with other queues. Weights default to 1, and a queue with weight 2 should receive approximately twice From 8a83bb7ad6177f473c20c4cc9c0f46746224332c Mon Sep 17 00:00:00 2001 From: Jing Zhao Date: Sat, 21 Jun 2014 22:59:34 +0000 Subject: [PATCH 017/112] HDFS-4667. Capture renamed files/directories in snapshot diff report. Contributed by Jing Zhao and Binglin Chang. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1604488 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../hdfs/protocol/SnapshotDiffReport.java | 58 +++- .../hadoop/hdfs/protocolPB/PBHelper.java | 28 +- .../hdfs/server/namenode/FSDirectory.java | 49 +-- .../hdfs/server/namenode/INodeDirectory.java | 8 +- .../namenode/INodeDirectoryAttributes.java | 6 +- .../hdfs/server/namenode/INodeFile.java | 9 + .../server/namenode/INodeFileAttributes.java | 11 + .../hdfs/server/namenode/INodeReference.java | 31 +- .../snapshot/AbstractINodeDiffList.java | 40 +-- .../DirectoryWithSnapshotFeature.java | 83 ++--- .../snapshot/FileWithSnapshotFeature.java | 37 ++- .../snapshot/INodeDirectorySnapshottable.java | 157 ++++++++-- .../hadoop-hdfs/src/main/proto/hdfs.proto | 1 + .../src/site/xdoc/HdfsSnapshots.xml | 28 ++ .../TestFullPathNameWithSnapshot.java | 295 ------------------ .../snapshot/TestRenameWithSnapshots.java | 75 ++--- .../snapshot/TestSnapshotDiffReport.java | 170 +++++++++- 18 files changed, 560 insertions(+), 529 deletions(-) delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFullPathNameWithSnapshot.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 5cbea0dd7cf..2ffbee85d27 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -458,6 +458,9 @@ Release 2.5.0 - UNRELEASED HDFS-6557. Move the reference of fsimage to FSNamesystem. (wheat9) + HDFS-4667. Capture renamed files/directories in snapshot diff report. (jing9 + and Binglin Chang via jing9) + OPTIMIZATIONS HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java index 265a05d08e3..ba96371ffa5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotDiffReport.java @@ -25,6 +25,8 @@ import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable.SnapshotDiffInfo; +import com.google.common.base.Objects; + /** * This class represents to end users the difference between two snapshots of * the same directory, or the difference between a snapshot of the directory and @@ -79,43 +81,64 @@ public static class DiffReportEntry { /** The type of the difference. */ private final DiffType type; /** - * The relative path (related to the snapshot root) of the file/directory - * where changes have happened + * The relative path (related to the snapshot root) of 1) the file/directory + * where changes have happened, or 2) the source file/dir of a rename op. */ - private final byte[] relativePath; + private final byte[] sourcePath; + private final byte[] targetPath; - public DiffReportEntry(DiffType type, byte[] path) { + public DiffReportEntry(DiffType type, byte[] sourcePath) { + this(type, sourcePath, null); + } + + public DiffReportEntry(DiffType type, byte[][] sourcePathComponents) { + this(type, sourcePathComponents, null); + } + + public DiffReportEntry(DiffType type, byte[] sourcePath, byte[] targetPath) { this.type = type; - this.relativePath = path; + this.sourcePath = sourcePath; + this.targetPath = targetPath; } - public DiffReportEntry(DiffType type, byte[][] pathComponents) { + public DiffReportEntry(DiffType type, byte[][] sourcePathComponents, + byte[][] targetPathComponents) { this.type = type; - this.relativePath = DFSUtil.byteArray2bytes(pathComponents); + this.sourcePath = DFSUtil.byteArray2bytes(sourcePathComponents); + this.targetPath = targetPathComponents == null ? null : DFSUtil + .byteArray2bytes(targetPathComponents); } @Override public String toString() { - return type.getLabel() + "\t" + getRelativePathString(); + String str = type.getLabel() + "\t" + getPathString(sourcePath); + if (type == DiffType.RENAME) { + str += " -> " + getPathString(targetPath); + } + return str; } public DiffType getType() { return type; } - public String getRelativePathString() { - String path = DFSUtil.bytes2String(relativePath); - if (path.isEmpty()) { + static String getPathString(byte[] path) { + String pathStr = DFSUtil.bytes2String(path); + if (pathStr.isEmpty()) { return Path.CUR_DIR; } else { - return Path.CUR_DIR + Path.SEPARATOR + path; + return Path.CUR_DIR + Path.SEPARATOR + pathStr; } } - public byte[] getRelativePath() { - return relativePath; + public byte[] getSourcePath() { + return sourcePath; } - + + public byte[] getTargetPath() { + return targetPath; + } + @Override public boolean equals(Object other) { if (this == other) { @@ -124,14 +147,15 @@ public boolean equals(Object other) { if (other != null && other instanceof DiffReportEntry) { DiffReportEntry entry = (DiffReportEntry) other; return type.equals(entry.getType()) - && Arrays.equals(relativePath, entry.getRelativePath()); + && Arrays.equals(sourcePath, entry.getSourcePath()) + && Arrays.equals(targetPath, entry.getTargetPath()); } return false; } @Override public int hashCode() { - return Arrays.hashCode(relativePath); + return Objects.hashCode(getSourcePath(), getTargetPath()); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java index f3b62c07f11..e9435024bed 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java @@ -44,7 +44,6 @@ import org.apache.hadoop.ha.proto.HAServiceProtocolProtos; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.StorageType; -import org.apache.hadoop.hdfs.XAttrHelper; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry; import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo; @@ -1737,24 +1736,29 @@ public static DiffReportEntry convert(SnapshotDiffReportEntryProto entry) { } DiffType type = DiffType.getTypeFromLabel(entry .getModificationLabel()); - return type == null ? null : - new DiffReportEntry(type, entry.getFullpath().toByteArray()); + return type == null ? null : new DiffReportEntry(type, entry.getFullpath() + .toByteArray(), entry.hasTargetPath() ? entry.getTargetPath() + .toByteArray() : null); } public static SnapshotDiffReportEntryProto convert(DiffReportEntry entry) { if (entry == null) { return null; } - byte[] fullPath = entry.getRelativePath(); - ByteString fullPathString = ByteString - .copyFrom(fullPath == null ? DFSUtil.EMPTY_BYTES : fullPath); - + ByteString sourcePath = ByteString + .copyFrom(entry.getSourcePath() == null ? DFSUtil.EMPTY_BYTES : entry + .getSourcePath()); String modification = entry.getType().getLabel(); - - SnapshotDiffReportEntryProto entryProto = SnapshotDiffReportEntryProto - .newBuilder().setFullpath(fullPathString) - .setModificationLabel(modification).build(); - return entryProto; + SnapshotDiffReportEntryProto.Builder builder = SnapshotDiffReportEntryProto + .newBuilder().setFullpath(sourcePath) + .setModificationLabel(modification); + if (entry.getType() == DiffType.RENAME) { + ByteString targetPath = ByteString + .copyFrom(entry.getTargetPath() == null ? DFSUtil.EMPTY_BYTES : entry + .getTargetPath()); + builder.setTargetPath(targetPath); + } + return builder.build(); } public static SnapshotDiffReport convert(SnapshotDiffReportProto reportProto) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index d00372957f9..d9e3be931d2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@ -2790,7 +2790,7 @@ void shutdown() { * Note that this method cannot handle scenarios where the inode is in a * snapshot. */ - static byte[][] getPathComponents(INode inode) { + public static byte[][] getPathComponents(INode inode) { List components = new ArrayList(); components.add(0, inode.getLocalNameBytes()); while(inode.getParent() != null) { @@ -2800,53 +2800,6 @@ static byte[][] getPathComponents(INode inode) { return components.toArray(new byte[components.size()][]); } - /** - * The same functionality with {@link #getPathComponents(INode)}, but can - * handle snapshots. - */ - public static byte[][] getPathComponentsWithSnapshot(INode inode) { - List components = new ArrayList(); - boolean inSnapshot = false; - int snapshotId = Snapshot.CURRENT_STATE_ID; - do { - if (inode instanceof INodeReference.WithCount) { - // identify the corresponding WithName or DstReference node - inode = ((WithCount) inode).getParentRef(snapshotId); - } else { // normal INode and WithName/DstReference - if (inode instanceof INodeDirectory - && inode.asDirectory().isSnapshottable() && inSnapshot - && snapshotId != Snapshot.CURRENT_STATE_ID) { - INodeDirectorySnapshottable sdir = (INodeDirectorySnapshottable) inode - .asDirectory(); - Snapshot snapshot = sdir.getSnapshotById(snapshotId); - if (snapshot != null) { - components.add(0, snapshot.getRoot().getLocalNameBytes()); - components.add(0, HdfsConstants.DOT_SNAPSHOT_DIR_BYTES); - // the snapshot has been found, thus no need to check snapshottable - // directory afterwards - inSnapshot = false; - } - } - INode parent = inode.getParentReference() != null ? inode - .getParentReference() : inode.getParent(); - if (parent != null && parent instanceof INodeDirectory) { - int sid = parent.asDirectory().searchChild(inode); - Preconditions.checkState(sid != Snapshot.NO_SNAPSHOT_ID); - if (sid != Snapshot.CURRENT_STATE_ID - && snapshotId == Snapshot.CURRENT_STATE_ID) { - snapshotId = sid; - inSnapshot = true; - } - components.add(0, inode.getLocalNameBytes()); - } else if (parent == null) { // root - components.add(0, inode.getLocalNameBytes()); - } - inode = parent; - } - } while (inode != null); - return components.toArray(new byte[components.size()][]); - } - /** * @return path components for reserved path, else null. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java index e5080a3c5ce..8781934bee6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java @@ -26,11 +26,9 @@ import java.util.Map; import org.apache.hadoop.fs.PathIsNotDirectoryException; -import org.apache.hadoop.fs.UnresolvedLinkException; import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.protocol.QuotaExceededException; -import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException; import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithCount; import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature; import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiffList; @@ -365,7 +363,7 @@ public INode getChild(byte[] name, int snapshotId) { * children list nor in any snapshot; otherwise the snapshot id of the * corresponding snapshot diff list. */ - int searchChild(INode inode) { + public int searchChild(INode inode) { INode child = getChild(inode.getLocalNameBytes(), Snapshot.CURRENT_STATE_ID); if (child != inode) { // inode is not in parent's children list, thus inode must be in @@ -764,7 +762,9 @@ public Quota.Counts cleanSubtree(final int snapshotId, int priorSnapshotId, public boolean metadataEquals(INodeDirectoryAttributes other) { return other != null && getQuotaCounts().equals(other.getQuotaCounts()) - && getPermissionLong() == other.getPermissionLong(); + && getPermissionLong() == other.getPermissionLong() + && getAclFeature() == other.getAclFeature() + && getXAttrFeature() == other.getXAttrFeature(); } /* diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryAttributes.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryAttributes.java index d95fa4651b2..b1e74859a9a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryAttributes.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryAttributes.java @@ -53,8 +53,10 @@ public Quota.Counts getQuotaCounts() { @Override public boolean metadataEquals(INodeDirectoryAttributes other) { return other != null - && this.getQuotaCounts().equals(other.getQuotaCounts()) - && getPermissionLong() == other.getPermissionLong(); + && getQuotaCounts().equals(other.getQuotaCounts()) + && getPermissionLong() == other.getPermissionLong() + && getAclFeature() == other.getAclFeature() + && getXAttrFeature() == other.getXAttrFeature(); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java index a38e8d9772f..42025348afc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java @@ -144,6 +144,15 @@ public final INodeFile asFile() { return this; } + @Override + public boolean metadataEquals(INodeFileAttributes other) { + return other != null + && getHeaderLong()== other.getHeaderLong() + && getPermissionLong() == other.getPermissionLong() + && getAclFeature() == other.getAclFeature() + && getXAttrFeature() == other.getXAttrFeature(); + } + /* Start of Under-Construction Feature */ /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java index 127642506ae..37bf0884cf1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java @@ -36,6 +36,8 @@ public interface INodeFileAttributes extends INodeAttributes { /** @return the header as a long. */ public long getHeaderLong(); + public boolean metadataEquals(INodeFileAttributes other); + /** A copy of the inode file attributes */ public static class SnapshotCopy extends INodeAttributes.SnapshotCopy implements INodeFileAttributes { @@ -70,5 +72,14 @@ public long getPreferredBlockSize() { public long getHeaderLong() { return header; } + + @Override + public boolean metadataEquals(INodeFileAttributes other) { + return other != null + && getHeaderLong()== other.getHeaderLong() + && getPermissionLong() == other.getPermissionLong() + && getAclFeature() == other.getAclFeature() + && getXAttrFeature() == other.getXAttrFeature(); + } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java index 461f075a0b7..ac0f19d0321 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java @@ -435,21 +435,28 @@ WithName getPriorWithName(WithName post) { } } + /** + * @return the WithName/DstReference node contained in the given snapshot. + */ public INodeReference getParentRef(int snapshotId) { - // when the given snapshotId is CURRENT_STATE_ID, it is possible that we - // do not know where the corresponding inode belongs, thus we simply - // return the last reference node - if (snapshotId == Snapshot.CURRENT_STATE_ID) { - return this.getParentReference() != null ? this.getParentReference() - : this.getLastWithName(); - } - // otherwise we search the withNameList - for (int i = 0; i < withNameList.size(); i++) { - if (snapshotId <= withNameList.get(i).lastSnapshotId) { - return withNameList.get(i); + int start = 0; + int end = withNameList.size() - 1; + while (start < end) { + int mid = start + (end - start) / 2; + int sid = withNameList.get(mid).lastSnapshotId; + if (sid == snapshotId) { + return withNameList.get(mid); + } else if (sid < snapshotId) { + start = mid + 1; + } else { + end = mid; } } - return this.getParentReference(); + if (withNameList.get(start).lastSnapshotId >= snapshotId) { + return withNameList.get(start); + } else { + return this.getParentReference(); + } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java index 3c16b2ed523..d918495765d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java @@ -227,32 +227,34 @@ public final int getSnapshotById(final int snapshotId) { D diff = getDiffById(snapshotId); return diff == null ? Snapshot.CURRENT_STATE_ID : diff.getSnapshotId(); } - - /** - * Check if changes have happened between two snapshots. - * @param earlier The snapshot taken earlier - * @param later The snapshot taken later - * @return Whether or not modifications (including diretory/file metadata - * change, file creation/deletion under the directory) have happened - * between snapshots. - */ - final boolean changedBetweenSnapshots(Snapshot earlier, Snapshot later) { + + final int[] changedBetweenSnapshots(Snapshot from, Snapshot to) { + Snapshot earlier = from; + Snapshot later = to; + if (Snapshot.ID_COMPARATOR.compare(from, to) > 0) { + earlier = to; + later = from; + } + final int size = diffs.size(); int earlierDiffIndex = Collections.binarySearch(diffs, earlier.getId()); + int laterDiffIndex = later == null ? size : Collections + .binarySearch(diffs, later.getId()); if (-earlierDiffIndex - 1 == size) { // if the earlierSnapshot is after the latest SnapshotDiff stored in // diffs, no modification happened after the earlierSnapshot - return false; + return null; } - if (later != null) { - int laterDiffIndex = Collections.binarySearch(diffs, later.getId()); - if (laterDiffIndex == -1 || laterDiffIndex == 0) { - // if the laterSnapshot is the earliest SnapshotDiff stored in diffs, or - // before it, no modification happened before the laterSnapshot - return false; - } + if (laterDiffIndex == -1 || laterDiffIndex == 0) { + // if the laterSnapshot is the earliest SnapshotDiff stored in diffs, or + // before it, no modification happened before the laterSnapshot + return null; } - return true; + earlierDiffIndex = earlierDiffIndex < 0 ? (-earlierDiffIndex - 1) + : earlierDiffIndex; + laterDiffIndex = laterDiffIndex < 0 ? (-laterDiffIndex - 1) + : laterDiffIndex; + return new int[]{earlierDiffIndex, laterDiffIndex}; } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java index c82309358e2..e6645da4f97 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.util.ArrayDeque; import java.util.ArrayList; -import java.util.Collections; import java.util.Deque; import java.util.HashMap; import java.util.Iterator; @@ -42,6 +41,7 @@ import org.apache.hadoop.hdfs.server.namenode.INodeFile; import org.apache.hadoop.hdfs.server.namenode.INodeReference; import org.apache.hadoop.hdfs.server.namenode.Quota; +import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable.SnapshotDiffInfo.RenameEntry; import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat.ReferenceMap; import org.apache.hadoop.hdfs.util.Diff; import org.apache.hadoop.hdfs.util.Diff.Container; @@ -170,49 +170,33 @@ private void getDirsInDeleted(List dirList) { * @return A list of {@link DiffReportEntry} as the diff report. */ public List generateReport(byte[][] parentPath, - boolean fromEarlier) { - List cList = new ArrayList(); - List dList = new ArrayList(); - int c = 0, d = 0; + boolean fromEarlier, Map renameMap) { + List list = new ArrayList(); List created = getList(ListType.CREATED); List deleted = getList(ListType.DELETED); byte[][] fullPath = new byte[parentPath.length + 1][]; System.arraycopy(parentPath, 0, fullPath, 0, parentPath.length); - for (; c < created.size() && d < deleted.size(); ) { - INode cnode = created.get(c); - INode dnode = deleted.get(d); - if (cnode.compareTo(dnode.getLocalNameBytes()) == 0) { + for (INode cnode : created) { + RenameEntry entry = renameMap.get(cnode.getId()); + if (entry == null || !entry.isRename()) { fullPath[fullPath.length - 1] = cnode.getLocalNameBytes(); - // must be the case: delete first and then create an inode with the - // same name - cList.add(new DiffReportEntry(DiffType.CREATE, fullPath)); - dList.add(new DiffReportEntry(DiffType.DELETE, fullPath)); - c++; - d++; - } else if (cnode.compareTo(dnode.getLocalNameBytes()) < 0) { - fullPath[fullPath.length - 1] = cnode.getLocalNameBytes(); - cList.add(new DiffReportEntry(fromEarlier ? DiffType.CREATE + list.add(new DiffReportEntry(fromEarlier ? DiffType.CREATE : DiffType.DELETE, fullPath)); - c++; - } else { - fullPath[fullPath.length - 1] = dnode.getLocalNameBytes(); - dList.add(new DiffReportEntry(fromEarlier ? DiffType.DELETE - : DiffType.CREATE, fullPath)); - d++; } } - for (; d < deleted.size(); d++) { - fullPath[fullPath.length - 1] = deleted.get(d).getLocalNameBytes(); - dList.add(new DiffReportEntry(fromEarlier ? DiffType.DELETE - : DiffType.CREATE, fullPath)); + for (INode dnode : deleted) { + RenameEntry entry = renameMap.get(dnode.getId()); + if (entry != null && entry.isRename()) { + list.add(new DiffReportEntry(DiffType.RENAME, + fromEarlier ? entry.getSourcePath() : entry.getTargetPath(), + fromEarlier ? entry.getTargetPath() : entry.getSourcePath())); + } else { + fullPath[fullPath.length - 1] = dnode.getLocalNameBytes(); + list.add(new DiffReportEntry(fromEarlier ? DiffType.DELETE + : DiffType.CREATE, fullPath)); + } } - for (; c < created.size(); c++) { - fullPath[fullPath.length - 1] = created.get(c).getLocalNameBytes(); - cList.add(new DiffReportEntry(fromEarlier ? DiffType.CREATE - : DiffType.DELETE, fullPath)); - } - dList.addAll(cList); - return dList; + return list; } } @@ -724,34 +708,21 @@ public void computeContentSummary4Snapshot(final Content.Counts counts) { */ boolean computeDiffBetweenSnapshots(Snapshot fromSnapshot, Snapshot toSnapshot, ChildrenDiff diff, INodeDirectory currentINode) { - Snapshot earlier = fromSnapshot; - Snapshot later = toSnapshot; - if (Snapshot.ID_COMPARATOR.compare(fromSnapshot, toSnapshot) > 0) { - earlier = toSnapshot; - later = fromSnapshot; - } - - boolean modified = diffs.changedBetweenSnapshots(earlier, later); - if (!modified) { + int[] diffIndexPair = diffs.changedBetweenSnapshots(fromSnapshot, + toSnapshot); + if (diffIndexPair == null) { return false; } - - final List difflist = diffs.asList(); - final int size = difflist.size(); - int earlierDiffIndex = Collections.binarySearch(difflist, earlier.getId()); - int laterDiffIndex = later == null ? size : Collections - .binarySearch(difflist, later.getId()); - earlierDiffIndex = earlierDiffIndex < 0 ? (-earlierDiffIndex - 1) - : earlierDiffIndex; - laterDiffIndex = laterDiffIndex < 0 ? (-laterDiffIndex - 1) - : laterDiffIndex; + int earlierDiffIndex = diffIndexPair[0]; + int laterDiffIndex = diffIndexPair[1]; boolean dirMetadataChanged = false; INodeDirectoryAttributes dirCopy = null; + List difflist = diffs.asList(); for (int i = earlierDiffIndex; i < laterDiffIndex; i++) { DirectoryDiff sdiff = difflist.get(i); diff.combinePosterior(sdiff.diff, null); - if (dirMetadataChanged == false && sdiff.snapshotINode != null) { + if (!dirMetadataChanged && sdiff.snapshotINode != null) { if (dirCopy == null) { dirCopy = sdiff.snapshotINode; } else if (!dirCopy.metadataEquals(sdiff.snapshotINode)) { @@ -763,7 +734,7 @@ boolean computeDiffBetweenSnapshots(Snapshot fromSnapshot, if (!diff.isEmpty() || dirMetadataChanged) { return true; } else if (dirCopy != null) { - for (int i = laterDiffIndex; i < size; i++) { + for (int i = laterDiffIndex; i < difflist.size(); i++) { if (!dirCopy.metadataEquals(difflist.get(i).snapshotINode)) { return true; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java index 52adfc6dd66..9e57b151954 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hdfs.server.namenode.INode; import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo; import org.apache.hadoop.hdfs.server.namenode.INodeFile; +import org.apache.hadoop.hdfs.server.namenode.INodeFileAttributes; import org.apache.hadoop.hdfs.server.namenode.Quota; /** @@ -73,7 +74,41 @@ public short getMaxBlockRepInDiffs() { } return max; } - + + boolean changedBetweenSnapshots(INodeFile file, Snapshot from, Snapshot to) { + int[] diffIndexPair = diffs.changedBetweenSnapshots(from, to); + if (diffIndexPair == null) { + return false; + } + int earlierDiffIndex = diffIndexPair[0]; + int laterDiffIndex = diffIndexPair[1]; + + final List diffList = diffs.asList(); + final long earlierLength = diffList.get(earlierDiffIndex).getFileSize(); + final long laterLength = laterDiffIndex == diffList.size() ? file + .computeFileSize(true, false) : diffList.get(laterDiffIndex) + .getFileSize(); + if (earlierLength != laterLength) { // file length has been changed + return true; + } + + INodeFileAttributes earlierAttr = null; // check the metadata + for (int i = earlierDiffIndex; i < laterDiffIndex; i++) { + FileDiff diff = diffList.get(i); + if (diff.snapshotINode != null) { + earlierAttr = diff.snapshotINode; + break; + } + } + if (earlierAttr == null) { // no meta-change at all, return false + return false; + } + INodeFileAttributes laterAttr = diffs.getSnapshotINode( + Math.max(Snapshot.getSnapshotId(from), Snapshot.getSnapshotId(to)), + file); + return !earlierAttr.metadataEquals(laterAttr); + } + public String getDetailedString() { return (isCurrentFileDeleted()? "(DELETED), ": ", ") + diffs; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java index 4288ec2afb0..eef92157091 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java @@ -24,6 +24,7 @@ import java.util.Comparator; import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.SortedMap; @@ -43,6 +44,9 @@ import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; import org.apache.hadoop.hdfs.server.namenode.INodeFile; import org.apache.hadoop.hdfs.server.namenode.INodeMap; +import org.apache.hadoop.hdfs.server.namenode.INodeReference; +import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithCount; +import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithName; import org.apache.hadoop.hdfs.server.namenode.Quota; import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.ChildrenDiff; import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiff; @@ -51,6 +55,7 @@ import org.apache.hadoop.util.Time; import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; import com.google.common.primitives.SignedBytes; /** @@ -98,7 +103,43 @@ public int compare(INode left, INode right) { } } }; - + + static class RenameEntry { + private byte[][] sourcePath; + private byte[][] targetPath; + + void setSource(INode source, byte[][] sourceParentPath) { + Preconditions.checkState(sourcePath == null); + sourcePath = new byte[sourceParentPath.length + 1][]; + System.arraycopy(sourceParentPath, 0, sourcePath, 0, + sourceParentPath.length); + sourcePath[sourcePath.length - 1] = source.getLocalNameBytes(); + } + + void setTarget(INode target, byte[][] targetParentPath) { + targetPath = new byte[targetParentPath.length + 1][]; + System.arraycopy(targetParentPath, 0, targetPath, 0, + targetParentPath.length); + targetPath[targetPath.length - 1] = target.getLocalNameBytes(); + } + + void setTarget(byte[][] targetPath) { + this.targetPath = targetPath; + } + + boolean isRename() { + return sourcePath != null && targetPath != null; + } + + byte[][] getSourcePath() { + return sourcePath; + } + + byte[][] getTargetPath() { + return targetPath; + } + } + /** The root directory of the snapshots */ private final INodeDirectorySnapshottable snapshotRoot; /** The starting point of the difference */ @@ -109,7 +150,7 @@ public int compare(INode left, INode right) { * A map recording modified INodeFile and INodeDirectory and their relative * path corresponding to the snapshot root. Sorted based on their names. */ - private final SortedMap diffMap = + private final SortedMap diffMap = new TreeMap(INODE_COMPARATOR); /** * A map capturing the detailed difference about file creation/deletion. @@ -119,7 +160,10 @@ public int compare(INode left, INode right) { */ private final Map dirDiffMap = new HashMap(); - + + private final Map renameMap = + new HashMap(); + SnapshotDiffInfo(INodeDirectorySnapshottable snapshotRoot, Snapshot start, Snapshot end) { this.snapshotRoot = snapshotRoot; @@ -132,8 +176,36 @@ private void addDirDiff(INodeDirectory dir, byte[][] relativePath, ChildrenDiff diff) { dirDiffMap.put(dir, diff); diffMap.put(dir, relativePath); + // detect rename + for (INode created : diff.getList(ListType.CREATED)) { + if (created.isReference()) { + RenameEntry entry = getEntry(created.getId()); + if (entry.getTargetPath() == null) { + entry.setTarget(created, relativePath); + } + } + } + for (INode deleted : diff.getList(ListType.DELETED)) { + if (deleted instanceof INodeReference.WithName) { + RenameEntry entry = getEntry(deleted.getId()); + entry.setSource(deleted, relativePath); + } + } } - + + private RenameEntry getEntry(long inodeId) { + RenameEntry entry = renameMap.get(inodeId); + if (entry == null) { + entry = new RenameEntry(); + renameMap.put(inodeId, entry); + } + return entry; + } + + private void setRenameTarget(long inodeId, byte[][] path) { + getEntry(inodeId).setTarget(path); + } + /** Add a modified file */ private void addFileDiff(INodeFile file, byte[][] relativePath) { diffMap.put(file, relativePath); @@ -152,11 +224,11 @@ public SnapshotDiffReport generateReport() { List diffReportList = new ArrayList(); for (INode node : diffMap.keySet()) { diffReportList.add(new DiffReportEntry(DiffType.MODIFY, diffMap - .get(node))); + .get(node), null)); if (node.isDirectory()) { ChildrenDiff dirDiff = dirDiffMap.get(node); List subList = dirDiff.generateReport( - diffMap.get(node), isFromEarlier()); + diffMap.get(node), isFromEarlier(), renameMap); diffReportList.addAll(subList); } } @@ -423,25 +495,37 @@ private Snapshot getSnapshotByName(String snapshotName) */ private void computeDiffRecursively(INode node, List parentPath, SnapshotDiffInfo diffReport) { - ChildrenDiff diff = new ChildrenDiff(); + final Snapshot earlierSnapshot = diffReport.isFromEarlier() ? + diffReport.from : diffReport.to; + final Snapshot laterSnapshot = diffReport.isFromEarlier() ? + diffReport.to : diffReport.from; byte[][] relativePath = parentPath.toArray(new byte[parentPath.size()][]); if (node.isDirectory()) { + final ChildrenDiff diff = new ChildrenDiff(); INodeDirectory dir = node.asDirectory(); DirectoryWithSnapshotFeature sf = dir.getDirectoryWithSnapshotFeature(); if (sf != null) { - boolean change = sf.computeDiffBetweenSnapshots(diffReport.from, - diffReport.to, diff, dir); + boolean change = sf.computeDiffBetweenSnapshots(earlierSnapshot, + laterSnapshot, diff, dir); if (change) { diffReport.addDirDiff(dir, relativePath, diff); } } - ReadOnlyList children = dir.getChildrenList( - diffReport.isFromEarlier() ? Snapshot.getSnapshotId(diffReport.to) : - Snapshot.getSnapshotId(diffReport.from)); + ReadOnlyList children = dir.getChildrenList(earlierSnapshot + .getId()); for (INode child : children) { final byte[] name = child.getLocalNameBytes(); - if (diff.searchIndex(ListType.CREATED, name) < 0 - && diff.searchIndex(ListType.DELETED, name) < 0) { + boolean toProcess = diff.searchIndex(ListType.DELETED, name) < 0; + if (!toProcess && child instanceof INodeReference.WithName) { + byte[][] renameTargetPath = findRenameTargetPath((WithName) child, + laterSnapshot == null ? Snapshot.CURRENT_STATE_ID : + laterSnapshot.getId()); + if (renameTargetPath != null) { + toProcess = true; + diffReport.setRenameTarget(child.getId(), renameTargetPath); + } + } + if (toProcess) { parentPath.add(name); computeDiffRecursively(child, parentPath, diffReport); parentPath.remove(parentPath.size() - 1); @@ -449,18 +533,47 @@ private void computeDiffRecursively(INode node, List parentPath, } } else if (node.isFile() && node.asFile().isWithSnapshot()) { INodeFile file = node.asFile(); - Snapshot earlierSnapshot = diffReport.isFromEarlier() ? diffReport.from - : diffReport.to; - Snapshot laterSnapshot = diffReport.isFromEarlier() ? diffReport.to - : diffReport.from; - boolean change = file.getDiffs().changedBetweenSnapshots(earlierSnapshot, - laterSnapshot); + boolean change = file.getFileWithSnapshotFeature() + .changedBetweenSnapshots(file, earlierSnapshot, laterSnapshot); if (change) { diffReport.addFileDiff(file, relativePath); } } } - + + /** + * We just found a deleted WithName node as the source of a rename operation. + * However, we should include it in our snapshot diff report as rename only + * if the rename target is also under the same snapshottable directory. + */ + private byte[][] findRenameTargetPath(INodeReference.WithName wn, + final int snapshotId) { + INode inode = wn.getReferredINode(); + final LinkedList ancestors = Lists.newLinkedList(); + while (inode != null) { + if (inode == this) { + return ancestors.toArray(new byte[ancestors.size()][]); + } + if (inode instanceof INodeReference.WithCount) { + inode = ((WithCount) inode).getParentRef(snapshotId); + } else { + INode parent = inode.getParentReference() != null ? inode + .getParentReference() : inode.getParent(); + if (parent != null && parent instanceof INodeDirectory) { + int sid = parent.asDirectory().searchChild(inode); + if (sid < snapshotId) { + return null; + } + } + if (!(parent instanceof WithCount)) { + ancestors.addFirst(inode.getLocalNameBytes()); + } + inode = parent; + } + } + return null; + } + /** * Replace itself with {@link INodeDirectoryWithSnapshot} or * {@link INodeDirectory} depending on the latest snapshot. @@ -549,4 +662,4 @@ public void remove() { }); } } -} +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto index 2d7ca245a73..6fbb5b34205 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto @@ -275,6 +275,7 @@ message SnapshottableDirectoryListingProto { message SnapshotDiffReportEntryProto { required bytes fullpath = 1; required string modificationLabel = 2; + optional bytes targetPath = 3; } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml b/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml index bd499c79c80..f809e855b92 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml @@ -255,7 +255,35 @@ fromSnapshotThe name of the starting snapshot. toSnapshotThe name of the ending snapshot.
    • +
  • Results: + + + + + +
    +The file/directory has been created.
    -The file/directory has been deleted.
    MThe file/directory has been modified.
    RThe file/directory has been renamed.
    +
    • +

    + A RENAME entry indicates a file/directory has been renamed but + is still under the same snapshottable directory. A file/directory is + reported as deleted if it was renamed to outside of the snapshottble directory. + A file/directory renamed from outside of the snapshottble directory is + reported as newly created. +

    +

    + The snapshot difference report does not guarantee the same operation sequence. + For example, if we rename the directory "/foo" to "/foo2", and + then append new data to the file "/foo2/bar", the difference report will + be: + + R. /foo -> /foo2 + M. /foo/bar + + I.e., the changes on the files/directories under a renamed directory is + reported using the original path before the rename ("/foo/bar" in + the above example). +

    See also the corresponding Java API SnapshotDiffReport getSnapshotDiffReport(Path path, String fromSnapshot, String toSnapshot) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFullPathNameWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFullPathNameWithSnapshot.java deleted file mode 100644 index 23a6adf2bc1..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFullPathNameWithSnapshot.java +++ /dev/null @@ -1,295 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs.server.namenode.snapshot; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.hdfs.DFSTestUtil; -import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.server.namenode.FSDirectory; -import org.apache.hadoop.hdfs.server.namenode.INode; -import org.apache.hadoop.hdfs.server.namenode.INodeReference; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -/** - * Test getting the full path name of a given inode. The INode may be in - * snapshot. - */ -public class TestFullPathNameWithSnapshot { - private static final int BLOCKSIZE = 1024; - private static final short REPLICATION = 1; - - private Configuration conf; - private MiniDFSCluster cluster; - private FSDirectory fsdir; - private DistributedFileSystem dfs; - - @Before - public void setUp() throws Exception { - conf = new Configuration(); - conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCKSIZE); - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(REPLICATION) - .build(); - cluster.waitActive(); - - fsdir = cluster.getNamesystem().getFSDirectory(); - dfs = cluster.getFileSystem(); - } - - @After - public void tearDown() throws Exception { - if (cluster != null) { - cluster.shutdown(); - } - } - - /** - * Normal case without snapshot involved - */ - @Test - public void testNormalINode() throws Exception { - final Path bar = new Path("/foo/bar"); - dfs.mkdirs(bar); - final Path file = new Path(bar, "file"); - DFSTestUtil.createFile(dfs, file, BLOCKSIZE, REPLICATION, 0L); - INode fileNode = fsdir.getINode4Write(file.toString()); - byte[][] pathComponents = FSDirectory - .getPathComponentsWithSnapshot(fileNode); - DFSTestUtil.checkComponentsEquals(INode.getPathComponents(file.toString()), - pathComponents); - } - - /** - * INode in deleted list - */ - @Test - public void testDeletedINode() throws Exception { - final Path foo = new Path("/foo"); - final Path f1 = new Path(foo, "f1"); - DFSTestUtil.createFile(dfs, f1, BLOCKSIZE, REPLICATION, 0L); - final Path bar = new Path(foo, "bar"); - dfs.mkdirs(bar); - final Path f2 = new Path(bar, "f2"); - DFSTestUtil.createFile(dfs, f2, BLOCKSIZE, REPLICATION, 0L); - - INode f1Node = fsdir.getINode4Write(f1.toString()); - INode f2Node = fsdir.getINode4Write(f2.toString()); - - SnapshotTestHelper.createSnapshot(dfs, foo, "s1"); - dfs.delete(bar, true); - SnapshotTestHelper.createSnapshot(dfs, foo, "s2"); - dfs.delete(f1, true); - - byte[][] f1Components = FSDirectory.getPathComponentsWithSnapshot(f1Node); - byte[][] f2Components = FSDirectory.getPathComponentsWithSnapshot(f2Node); - // expected: /foo/.snapshot/s2/f1 - String f1Snapshot = SnapshotTestHelper.getSnapshotPath(foo, "s2", - f1.getName()).toString(); - // expected: /foo/.snapshot/s1/bar/f2 - String f2Snapshot = SnapshotTestHelper.getSnapshotPath(foo, "s1", "bar/f2") - .toString(); - DFSTestUtil.checkComponentsEquals(INode.getPathComponents(f1Snapshot), - f1Components); - DFSTestUtil.checkComponentsEquals(INode.getPathComponents(f2Snapshot), - f2Components); - - // delete snapshot s2 - dfs.deleteSnapshot(foo, "s2"); - // expected: /foo/.snapshot/s1/f1 - f1Snapshot = SnapshotTestHelper.getSnapshotPath(foo, "s1", f1.getName()) - .toString(); - f1Components = FSDirectory.getPathComponentsWithSnapshot(f1Node); - DFSTestUtil.checkComponentsEquals(INode.getPathComponents(f1Snapshot), - f1Components); - } - - /** - * INode after renaming - */ - @Test - public void testRenamedINode() throws Exception { - final Path foo = new Path("/foo"); - final Path bar = new Path(foo, "bar"); - final Path f1 = new Path(bar, "f1"); - final Path f2 = new Path(bar, "f2"); - DFSTestUtil.createFile(dfs, f1, BLOCKSIZE, REPLICATION, 0L); - DFSTestUtil.createFile(dfs, f2, BLOCKSIZE, REPLICATION, 0L); - - // create snapshot s1 - SnapshotTestHelper.createSnapshot(dfs, foo, "s1"); - INode f2Node = fsdir.getINode4Write(f2.toString()); - // delete /foo/bar/f2 - dfs.delete(f2, true); - // rename bar to bar2 - final Path bar2 = new Path(foo, "bar2"); - dfs.rename(bar, bar2); - // create snapshot s2 - SnapshotTestHelper.createSnapshot(dfs, foo, "s2"); - - // /foo/.snapshot/s1/bar - Path barPath = SnapshotTestHelper.getSnapshotPath(foo, "s1", bar.getName()); - INode barNode = fsdir.getINode(barPath.toString()); - Assert.assertTrue(barNode instanceof INodeReference.WithName); - INode bar2Node = fsdir.getINode(bar2.toString()); - Assert.assertTrue(bar2Node instanceof INodeReference.DstReference); - byte[][] barComponents = FSDirectory.getPathComponentsWithSnapshot(barNode); - byte[][] bar2Components = FSDirectory - .getPathComponentsWithSnapshot(bar2Node); - DFSTestUtil.checkComponentsEquals( - INode.getPathComponents(barPath.toString()), barComponents); - DFSTestUtil.checkComponentsEquals(INode.getPathComponents(bar2.toString()), - bar2Components); - - byte[][] f2Components = FSDirectory.getPathComponentsWithSnapshot(f2Node); - // expected: /foo/.snapshot/s1/bar/f2 - Path deletedf2 = SnapshotTestHelper.getSnapshotPath(foo, "s1", "bar/f2"); - DFSTestUtil.checkComponentsEquals( - INode.getPathComponents(deletedf2.toString()), f2Components); - - final Path newf1 = new Path(bar2, f1.getName()); - INode f1Node = fsdir.getINode(newf1.toString()); - Assert.assertTrue(dfs.delete(newf1, true)); - Path deletedf1 = SnapshotTestHelper.getSnapshotPath(foo, "s2", "bar2/f1"); - byte[][] f1Components = FSDirectory.getPathComponentsWithSnapshot(f1Node); - DFSTestUtil.checkComponentsEquals( - INode.getPathComponents(deletedf1.toString()), f1Components); - } - - /** - * Similar with testRenamedINode but the rename is across two snapshottable - * directory. - */ - @Test - public void testRenamedINode2() throws Exception { - final Path foo1 = new Path("/foo1"); - final Path foo2 = new Path("/foo2"); - final Path bar = new Path(foo1, "bar"); - final Path f1 = new Path(bar, "f1"); - final Path f2 = new Path(bar, "f2"); - dfs.mkdirs(foo2); - DFSTestUtil.createFile(dfs, f1, BLOCKSIZE, REPLICATION, 0L); - DFSTestUtil.createFile(dfs, f2, BLOCKSIZE, REPLICATION, 0L); - - // create snapshots on foo1 and foo2 - SnapshotTestHelper.createSnapshot(dfs, foo1, "s1"); - SnapshotTestHelper.createSnapshot(dfs, foo2, "s2"); - INode f2Node = fsdir.getINode4Write(f2.toString()); - // delete /foo1/bar/f2 - dfs.delete(f2, true); - // rename bar to bar2 - final Path bar2 = new Path(foo2, "bar2"); - dfs.rename(bar, bar2); - // create snapshot s3 and s4 on foo1 and foo2 - SnapshotTestHelper.createSnapshot(dfs, foo1, "s3"); - SnapshotTestHelper.createSnapshot(dfs, foo2, "s4"); - - // /foo1/.snapshot/s1/bar - Path barPath = SnapshotTestHelper - .getSnapshotPath(foo1, "s1", bar.getName()); - INode barNode = fsdir.getINode(barPath.toString()); - Assert.assertTrue(barNode instanceof INodeReference.WithName); - INode bar2Node = fsdir.getINode(bar2.toString()); - Assert.assertTrue(bar2Node instanceof INodeReference.DstReference); - byte[][] barComponents = FSDirectory.getPathComponentsWithSnapshot(barNode); - byte[][] bar2Components = FSDirectory - .getPathComponentsWithSnapshot(bar2Node); - DFSTestUtil.checkComponentsEquals( - INode.getPathComponents(barPath.toString()), barComponents); - DFSTestUtil.checkComponentsEquals(INode.getPathComponents(bar2.toString()), - bar2Components); - - byte[][] f2Components = FSDirectory.getPathComponentsWithSnapshot(f2Node); - // expected: /foo1/.snapshot/s1/bar/f2 - Path deletedf2 = SnapshotTestHelper.getSnapshotPath(foo1, "s1", "bar/f2"); - DFSTestUtil.checkComponentsEquals( - INode.getPathComponents(deletedf2.toString()), f2Components); - - final Path newf1 = new Path(bar2, f1.getName()); - INode f1Node = fsdir.getINode(newf1.toString()); - Assert.assertTrue(dfs.delete(newf1, true)); - // /foo2/.snapshot/s4/bar2/f1 - Path deletedf1 = SnapshotTestHelper.getSnapshotPath(foo2, "s4", "bar2/f1"); - byte[][] f1Components = FSDirectory.getPathComponentsWithSnapshot(f1Node); - DFSTestUtil.checkComponentsEquals( - INode.getPathComponents(deletedf1.toString()), f1Components); - } - - /** - * Rename a directory to its prior descendant - */ - @Test - public void testNestedRename() throws Exception { - final Path sdir1 = new Path("/dir1"); - final Path sdir2 = new Path("/dir2"); - final Path foo = new Path(sdir1, "foo"); - final Path bar = new Path(foo, "bar"); - dfs.mkdirs(bar); - dfs.mkdirs(sdir2); - - SnapshotTestHelper.createSnapshot(dfs, sdir1, "s1"); - - // /dir1/foo/bar -> /dir2/bar - final Path bar2 = new Path(sdir2, "bar"); - dfs.rename(bar, bar2); - - // /dir1/foo -> /dir2/bar/foo - final Path foo2 = new Path(bar2, "foo"); - dfs.rename(foo, foo2); - - // /dir2/bar - INode bar2Node = fsdir.getINode(bar2.toString()); - Assert.assertTrue(bar2Node instanceof INodeReference.DstReference); - byte[][] bar2Components = FSDirectory - .getPathComponentsWithSnapshot(bar2Node); - DFSTestUtil.checkComponentsEquals(INode.getPathComponents(bar2.toString()), - bar2Components); - - // /dir1/.snapshot/s1/foo/bar - String oldbar = SnapshotTestHelper.getSnapshotPath(sdir1, "s1", "foo/bar") - .toString(); - INode oldbarNode = fsdir.getINode(oldbar); - Assert.assertTrue(oldbarNode instanceof INodeReference.WithName); - byte[][] oldbarComponents = FSDirectory - .getPathComponentsWithSnapshot(oldbarNode); - DFSTestUtil.checkComponentsEquals(INode.getPathComponents(oldbar), - oldbarComponents); - - // /dir2/bar/foo - INode foo2Node = fsdir.getINode(foo2.toString()); - Assert.assertTrue(foo2Node instanceof INodeReference.DstReference); - byte[][] foo2Components = FSDirectory - .getPathComponentsWithSnapshot(foo2Node); - DFSTestUtil.checkComponentsEquals(INode.getPathComponents(foo2.toString()), - foo2Components); - - // /dir1/.snapshot/s1/foo - String oldfoo = SnapshotTestHelper.getSnapshotPath(sdir1, "s1", - foo.getName()).toString(); - INode oldfooNode = fsdir.getINode(oldfoo); - Assert.assertTrue(oldfooNode instanceof INodeReference.WithName); - byte[][] oldfooComponents = FSDirectory - .getPathComponentsWithSnapshot(oldfooNode); - DFSTestUtil.checkComponentsEquals(INode.getPathComponents(oldfoo), - oldfooComponents); - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java index c88aaf2410e..2d1fa77ace7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java @@ -169,10 +169,11 @@ public void testRenameFromSDir2NonSDir() throws Exception { } private static boolean existsInDiffReport(List entries, - DiffType type, String relativePath) { + DiffType type, String sourcePath, String targetPath) { for (DiffReportEntry entry : entries) { - if ((entry.getType() == type) - && ((new String(entry.getRelativePath())).compareTo(relativePath) == 0)) { + if (entry.equals(new DiffReportEntry(type, DFSUtil + .string2Bytes(sourcePath), targetPath == null ? null : DFSUtil + .string2Bytes(targetPath)))) { return true; } } @@ -195,8 +196,9 @@ public void testRenameFileNotInSnapshot() throws Exception { SnapshotDiffReport diffReport = hdfs.getSnapshotDiffReport(sub1, snap1, ""); List entries = diffReport.getDiffList(); assertTrue(entries.size() == 2); - assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "")); - assertTrue(existsInDiffReport(entries, DiffType.CREATE, file2.getName())); + assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null)); + assertTrue(existsInDiffReport(entries, DiffType.CREATE, file2.getName(), + null)); } /** @@ -215,10 +217,10 @@ public void testRenameFileInSnapshot() throws Exception { SnapshotDiffReport diffReport = hdfs.getSnapshotDiffReport(sub1, snap1, ""); System.out.println("DiffList is " + diffReport.toString()); List entries = diffReport.getDiffList(); - assertTrue(entries.size() == 3); - assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "")); - assertTrue(existsInDiffReport(entries, DiffType.CREATE, file2.getName())); - assertTrue(existsInDiffReport(entries, DiffType.DELETE, file1.getName())); + assertTrue(entries.size() == 2); + assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null)); + assertTrue(existsInDiffReport(entries, DiffType.RENAME, file1.getName(), + file2.getName())); } @Test (timeout=60000) @@ -238,26 +240,26 @@ public void testRenameTwiceInSnapshot() throws Exception { diffReport = hdfs.getSnapshotDiffReport(sub1, snap1, snap2); LOG.info("DiffList is " + diffReport.toString()); List entries = diffReport.getDiffList(); - assertTrue(entries.size() == 3); - assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "")); - assertTrue(existsInDiffReport(entries, DiffType.CREATE, file2.getName())); - assertTrue(existsInDiffReport(entries, DiffType.DELETE, file1.getName())); + assertTrue(entries.size() == 2); + assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null)); + assertTrue(existsInDiffReport(entries, DiffType.RENAME, file1.getName(), + file2.getName())); diffReport = hdfs.getSnapshotDiffReport(sub1, snap2, ""); LOG.info("DiffList is " + diffReport.toString()); entries = diffReport.getDiffList(); - assertTrue(entries.size() == 3); - assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "")); - assertTrue(existsInDiffReport(entries, DiffType.CREATE, file3.getName())); - assertTrue(existsInDiffReport(entries, DiffType.DELETE, file2.getName())); + assertTrue(entries.size() == 2); + assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null)); + assertTrue(existsInDiffReport(entries, DiffType.RENAME, file2.getName(), + file3.getName())); diffReport = hdfs.getSnapshotDiffReport(sub1, snap1, ""); LOG.info("DiffList is " + diffReport.toString()); entries = diffReport.getDiffList(); - assertTrue(entries.size() == 3); - assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "")); - assertTrue(existsInDiffReport(entries, DiffType.CREATE, file3.getName())); - assertTrue(existsInDiffReport(entries, DiffType.DELETE, file1.getName())); + assertTrue(entries.size() == 2); + assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null)); + assertTrue(existsInDiffReport(entries, DiffType.RENAME, file1.getName(), + file3.getName())); } @Test (timeout=60000) @@ -280,11 +282,10 @@ public void testRenameFileInSubDirOfDirWithSnapshot() throws Exception { ""); LOG.info("DiffList is \n\"" + diffReport.toString() + "\""); List entries = diffReport.getDiffList(); - assertTrue(existsInDiffReport(entries, DiffType.MODIFY, sub2.getName())); - assertTrue(existsInDiffReport(entries, DiffType.CREATE, sub2.getName() - + "/" + sub2file2.getName())); - assertTrue(existsInDiffReport(entries, DiffType.DELETE, sub2.getName() - + "/" + sub2file1.getName())); + assertTrue(existsInDiffReport(entries, DiffType.MODIFY, sub2.getName(), + null)); + assertTrue(existsInDiffReport(entries, DiffType.RENAME, sub2.getName() + + "/" + sub2file1.getName(), sub2.getName() + "/" + sub2file2.getName())); } @Test (timeout=60000) @@ -307,10 +308,10 @@ public void testRenameDirectoryInSnapshot() throws Exception { ""); LOG.info("DiffList is \n\"" + diffReport.toString() + "\""); List entries = diffReport.getDiffList(); - assertEquals(3, entries.size()); - assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "")); - assertTrue(existsInDiffReport(entries, DiffType.CREATE, sub3.getName())); - assertTrue(existsInDiffReport(entries, DiffType.DELETE, sub2.getName())); + assertEquals(2, entries.size()); + assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null)); + assertTrue(existsInDiffReport(entries, DiffType.RENAME, sub2.getName(), + sub3.getName())); } /** @@ -2406,12 +2407,12 @@ public void testRenameWithOverWrite() throws Exception { LOG.info("DiffList is \n\"" + report.toString() + "\""); List entries = report.getDiffList(); assertEquals(7, entries.size()); - assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "")); - assertTrue(existsInDiffReport(entries, DiffType.MODIFY, foo.getName())); - assertTrue(existsInDiffReport(entries, DiffType.DELETE, bar.getName())); - assertTrue(existsInDiffReport(entries, DiffType.CREATE, newDir.getName())); - assertTrue(existsInDiffReport(entries, DiffType.DELETE, "foo/file1")); - assertTrue(existsInDiffReport(entries, DiffType.DELETE, "foo/file2")); - assertTrue(existsInDiffReport(entries, DiffType.DELETE, "foo/file3")); + assertTrue(existsInDiffReport(entries, DiffType.MODIFY, "", null)); + assertTrue(existsInDiffReport(entries, DiffType.MODIFY, foo.getName(), null)); + assertTrue(existsInDiffReport(entries, DiffType.MODIFY, bar.getName(), null)); + assertTrue(existsInDiffReport(entries, DiffType.DELETE, "foo/file1", null)); + assertTrue(existsInDiffReport(entries, DiffType.RENAME, "bar", "newDir")); + assertTrue(existsInDiffReport(entries, DiffType.RENAME, "foo/file2", "newDir/file2")); + assertTrue(existsInDiffReport(entries, DiffType.RENAME, "foo/file3", "newDir/file1")); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java index 8ba3ef4ecfc..0cc318b5ad2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java @@ -25,6 +25,7 @@ import java.util.HashMap; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Options.Rename; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DFSUtil; @@ -143,7 +144,7 @@ private void modifyAndCreateSnapshot(Path modifyDir, Path[] snapshotDirs) hdfs.createSnapshot(snapshotDir, genSnapshotName(snapshotDir)); } // modify file10 - hdfs.setReplication(file10, (short) (REPLICATION - 1)); + hdfs.setReplication(file10, (short) (REPLICATION + 1)); } /** check the correctness of the diff reports */ @@ -166,11 +167,11 @@ private void verifyDiffReport(Path dir, String from, String to, } else if (entry.getType() == DiffType.DELETE) { assertTrue(report.getDiffList().contains(entry)); assertTrue(inverseReport.getDiffList().contains( - new DiffReportEntry(DiffType.CREATE, entry.getRelativePath()))); + new DiffReportEntry(DiffType.CREATE, entry.getSourcePath()))); } else if (entry.getType() == DiffType.CREATE) { assertTrue(report.getDiffList().contains(entry)); assertTrue(inverseReport.getDiffList().contains( - new DiffReportEntry(DiffType.DELETE, entry.getRelativePath()))); + new DiffReportEntry(DiffType.DELETE, entry.getSourcePath()))); } } } @@ -329,5 +330,166 @@ public void testDiffReport2() throws Exception { new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("subsub1"))); } - + + /** + * Rename a directory to its prior descendant, and verify the diff report. + */ + @Test + public void testDiffReportWithRename() throws Exception { + final Path root = new Path("/"); + final Path sdir1 = new Path(root, "dir1"); + final Path sdir2 = new Path(root, "dir2"); + final Path foo = new Path(sdir1, "foo"); + final Path bar = new Path(foo, "bar"); + hdfs.mkdirs(bar); + hdfs.mkdirs(sdir2); + + // create snapshot on root + SnapshotTestHelper.createSnapshot(hdfs, root, "s1"); + + // /dir1/foo/bar -> /dir2/bar + final Path bar2 = new Path(sdir2, "bar"); + hdfs.rename(bar, bar2); + + // /dir1/foo -> /dir2/bar/foo + final Path foo2 = new Path(bar2, "foo"); + hdfs.rename(foo, foo2); + + SnapshotTestHelper.createSnapshot(hdfs, root, "s2"); + // let's delete /dir2 to make things more complicated + hdfs.delete(sdir2, true); + + verifyDiffReport(root, "s1", "s2", + new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), + new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir1")), + new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("dir1/foo"), + DFSUtil.string2Bytes("dir2/bar/foo")), + new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir2")), + new DiffReportEntry(DiffType.MODIFY, + DFSUtil.string2Bytes("dir1/foo/bar")), + new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir1/foo")), + new DiffReportEntry(DiffType.RENAME, DFSUtil + .string2Bytes("dir1/foo/bar"), DFSUtil.string2Bytes("dir2/bar"))); + } + + /** + * Rename a file/dir outside of the snapshottable dir should be reported as + * deleted. Rename a file/dir from outside should be reported as created. + */ + @Test + public void testDiffReportWithRenameOutside() throws Exception { + final Path root = new Path("/"); + final Path dir1 = new Path(root, "dir1"); + final Path dir2 = new Path(root, "dir2"); + final Path foo = new Path(dir1, "foo"); + final Path fileInFoo = new Path(foo, "file"); + final Path bar = new Path(dir2, "bar"); + final Path fileInBar = new Path(bar, "file"); + DFSTestUtil.createFile(hdfs, fileInFoo, BLOCKSIZE, REPLICATION, seed); + DFSTestUtil.createFile(hdfs, fileInBar, BLOCKSIZE, REPLICATION, seed); + + // create snapshot on /dir1 + SnapshotTestHelper.createSnapshot(hdfs, dir1, "s0"); + + // move bar into dir1 + final Path newBar = new Path(dir1, "newBar"); + hdfs.rename(bar, newBar); + // move foo out of dir1 into dir2 + final Path newFoo = new Path(dir2, "new"); + hdfs.rename(foo, newFoo); + + SnapshotTestHelper.createSnapshot(hdfs, dir1, "s1"); + verifyDiffReport(dir1, "s0", "s1", + new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), + new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes(newBar + .getName())), + new DiffReportEntry(DiffType.DELETE, + DFSUtil.string2Bytes(foo.getName()))); + } + + /** + * Renaming a file/dir then delete the ancestor dir of the rename target + * should be reported as deleted. + */ + @Test + public void testDiffReportWithRenameAndDelete() throws Exception { + final Path root = new Path("/"); + final Path dir1 = new Path(root, "dir1"); + final Path dir2 = new Path(root, "dir2"); + final Path foo = new Path(dir1, "foo"); + final Path fileInFoo = new Path(foo, "file"); + final Path bar = new Path(dir2, "bar"); + final Path fileInBar = new Path(bar, "file"); + DFSTestUtil.createFile(hdfs, fileInFoo, BLOCKSIZE, REPLICATION, seed); + DFSTestUtil.createFile(hdfs, fileInBar, BLOCKSIZE, REPLICATION, seed); + + SnapshotTestHelper.createSnapshot(hdfs, root, "s0"); + hdfs.rename(fileInFoo, fileInBar, Rename.OVERWRITE); + SnapshotTestHelper.createSnapshot(hdfs, root, "s1"); + verifyDiffReport(root, "s0", "s1", + new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), + new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir1/foo")), + new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir2/bar")), + new DiffReportEntry(DiffType.DELETE, DFSUtil + .string2Bytes("dir2/bar/file")), + new DiffReportEntry(DiffType.RENAME, + DFSUtil.string2Bytes("dir1/foo/file"), + DFSUtil.string2Bytes("dir2/bar/file"))); + + // delete bar + hdfs.delete(bar, true); + SnapshotTestHelper.createSnapshot(hdfs, root, "s2"); + verifyDiffReport(root, "s0", "s2", + new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), + new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir1/foo")), + new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("dir2")), + new DiffReportEntry(DiffType.DELETE, DFSUtil.string2Bytes("dir2/bar")), + new DiffReportEntry(DiffType.DELETE, + DFSUtil.string2Bytes("dir1/foo/file"))); + } + + @Test + public void testDiffReportWithRenameToNewDir() throws Exception { + final Path root = new Path("/"); + final Path foo = new Path(root, "foo"); + final Path fileInFoo = new Path(foo, "file"); + DFSTestUtil.createFile(hdfs, fileInFoo, BLOCKSIZE, REPLICATION, seed); + + SnapshotTestHelper.createSnapshot(hdfs, root, "s0"); + final Path bar = new Path(root, "bar"); + hdfs.mkdirs(bar); + final Path fileInBar = new Path(bar, "file"); + hdfs.rename(fileInFoo, fileInBar); + SnapshotTestHelper.createSnapshot(hdfs, root, "s1"); + + verifyDiffReport(root, "s0", "s1", + new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), + new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("foo")), + new DiffReportEntry(DiffType.CREATE, DFSUtil.string2Bytes("bar")), + new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("foo/file"), + DFSUtil.string2Bytes("bar/file"))); + } + + /** + * Rename a file and then append some data to it + */ + @Test + public void testDiffReportWithRenameAndAppend() throws Exception { + final Path root = new Path("/"); + final Path foo = new Path(root, "foo"); + DFSTestUtil.createFile(hdfs, foo, BLOCKSIZE, REPLICATION, seed); + + SnapshotTestHelper.createSnapshot(hdfs, root, "s0"); + final Path bar = new Path(root, "bar"); + hdfs.rename(foo, bar); + DFSTestUtil.appendFile(hdfs, bar, 10); // append 10 bytes + SnapshotTestHelper.createSnapshot(hdfs, root, "s1"); + + // we always put modification on the file before rename + verifyDiffReport(root, "s0", "s1", + new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("")), + new DiffReportEntry(DiffType.MODIFY, DFSUtil.string2Bytes("foo")), + new DiffReportEntry(DiffType.RENAME, DFSUtil.string2Bytes("foo"), + DFSUtil.string2Bytes("bar"))); + } } \ No newline at end of file From 1e89eba47d0f291b33fc26f9406231fc70b63a87 Mon Sep 17 00:00:00 2001 From: Haohui Mai Date: Sun, 22 Jun 2014 07:39:36 +0000 Subject: [PATCH 018/112] HDFS-6583. Remove clientNode in FileUnderConstructionFeature. Contributed by Haohui Mai. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1604541 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 1 + .../hdfs/server/namenode/FSDirectory.java | 8 +++--- .../hdfs/server/namenode/FSEditLogLoader.java | 3 +-- .../hdfs/server/namenode/FSImageFormat.java | 5 ++-- .../server/namenode/FSImageFormatPBINode.java | 4 +-- .../server/namenode/FSImageSerialization.java | 2 +- .../hdfs/server/namenode/FSNamesystem.java | 27 +++++++++---------- .../FileUnderConstructionFeature.java | 11 +------- .../hdfs/server/namenode/INodeFile.java | 6 ++--- .../hdfs/server/namenode/CreateEditsLog.java | 4 +-- .../hdfs/server/namenode/TestEditLog.java | 2 +- .../hdfs/server/namenode/TestINodeFile.java | 7 ++--- 12 files changed, 30 insertions(+), 50 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 2ffbee85d27..b46aa576cbd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -468,6 +468,7 @@ Release 2.5.0 - UNRELEASED HDFS-6460. Ignore stale and decommissioned nodes in NetworkTopology#sortByDistance. (Yongjun Zhang via wang) + HDFS-6583. Remove clientNode in FileUnderConstructionFeature. (wheat9) BUG FIXES HDFS-6112. NFS Gateway docs are incorrect for allowed hosts configuration. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index d9e3be931d2..488084c484d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@ -258,8 +258,8 @@ void disableQuotaChecks() { * @throws SnapshotAccessControlException */ INodeFile addFile(String path, PermissionStatus permissions, - short replication, long preferredBlockSize, String clientName, - String clientMachine, DatanodeDescriptor clientNode) + short replication, long preferredBlockSize, + String clientName, String clientMachine) throws FileAlreadyExistsException, QuotaExceededException, UnresolvedLinkException, SnapshotAccessControlException, AclException { @@ -267,7 +267,7 @@ INodeFile addFile(String path, PermissionStatus permissions, INodeFile newNode = new INodeFile(namesystem.allocateNewInodeId(), null, permissions, modTime, modTime, BlockInfo.EMPTY_ARRAY, replication, preferredBlockSize); - newNode.toUnderConstruction(clientName, clientMachine, clientNode); + newNode.toUnderConstruction(clientName, clientMachine); boolean added = false; writeLock(); @@ -305,7 +305,7 @@ INodeFile unprotectedAddFile( long id, newNode = new INodeFile(id, null, permissions, modificationTime, modificationTime, BlockInfo.EMPTY_ARRAY, replication, preferredBlockSize); - newNode.toUnderConstruction(clientName, clientMachine, null); + newNode.toUnderConstruction(clientName, clientMachine); } else { newNode = new INodeFile(id, null, permissions, modificationTime, atime, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java index 04785c231e7..0d4f48beaca 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java @@ -376,8 +376,7 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir, "for append"); } LocatedBlock lb = fsNamesys.prepareFileForWrite(path, - oldFile, addCloseOp.clientName, addCloseOp.clientMachine, null, - false, iip.getLatestSnapshotId(), false); + oldFile, addCloseOp.clientName, addCloseOp.clientMachine, false, iip.getLatestSnapshotId(), false); newFile = INodeFile.valueOf(fsDir.getINode(path), path, true); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java index 65d8bc9a72d..2beac45f363 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java @@ -781,7 +781,7 @@ INode loadINode(final byte[] localName, boolean isSnapshotINode, final INodeFile file = new INodeFile(inodeId, localName, permissions, modificationTime, atime, blocks, replication, blockSize); if (underConstruction) { - file.toUnderConstruction(clientName, clientMachine, null); + file.toUnderConstruction(clientName, clientMachine); } return fileDiffs == null ? file : new INodeFile(file, fileDiffs); } else if (numBlocks == -1) { @@ -933,8 +933,7 @@ LayoutVersion.Feature.ADD_INODE_ID, getLayoutVersion())) { } FileUnderConstructionFeature uc = cons.getFileUnderConstructionFeature(); - oldnode.toUnderConstruction(uc.getClientName(), uc.getClientMachine(), - uc.getClientNode()); + oldnode.toUnderConstruction(uc.getClientName(), uc.getClientMachine()); if (oldnode.numBlocks() > 0) { BlockInfo ucBlock = cons.getLastBlock(); // we do not replace the inode, just replace the last block of oldnode diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java index 077570e8eee..feff70465fd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java @@ -53,7 +53,6 @@ import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.XAttrCompactProto; import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.XAttrFeatureProto; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; -import org.apache.hadoop.hdfs.server.namenode.XAttrFeature; import org.apache.hadoop.hdfs.util.ReadOnlyList; import com.google.common.base.Preconditions; @@ -299,8 +298,7 @@ private INodeFile loadINodeFile(INodeSection.INode n) { // under-construction information if (f.hasFileUC()) { INodeSection.FileUnderConstructionFeature uc = f.getFileUC(); - file.toUnderConstruction(uc.getClientName(), uc.getClientMachine(), - null); + file.toUnderConstruction(uc.getClientName(), uc.getClientMachine()); if (blocks.length > 0) { BlockInfo lastBlk = file.getLastBlock(); // replace the last block of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java index 4429c528f65..314d55f9bfb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java @@ -149,7 +149,7 @@ static INodeFile readINodeUnderConstruction( INodeFile file = new INodeFile(inodeId, name, perm, modificationTime, modificationTime, blocks, blockReplication, preferredBlockSize); - file.toUnderConstruction(clientName, clientMachine, null); + file.toUnderConstruction(clientName, clientMachine); return file; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index a66578f4dfb..c570b1c54e3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -2388,9 +2388,6 @@ private void startFileInternal(FSPermissionChecker pc, String src, } checkFsObjectLimit(); - final DatanodeDescriptor clientNode = - blockManager.getDatanodeManager().getDatanodeByHost(clientMachine); - INodeFile newNode = null; // Always do an implicit mkdirs for parent directory tree. @@ -2398,7 +2395,7 @@ private void startFileInternal(FSPermissionChecker pc, String src, if (parent != null && mkdirsRecursively(parent.toString(), permissions, true, now())) { newNode = dir.addFile(src, permissions, replication, blockSize, - holder, clientMachine, clientNode); + holder, clientMachine); } if (newNode == null) { @@ -2473,10 +2470,8 @@ private LocatedBlock appendFileInternal(FSPermissionChecker pc, String src, throw new IOException("append: lastBlock=" + lastBlock + " of src=" + src + " is not sufficiently replicated yet."); } - final DatanodeDescriptor clientNode = - blockManager.getDatanodeManager().getDatanodeByHost(clientMachine); - return prepareFileForWrite(src, myFile, holder, clientMachine, clientNode, - true, iip.getLatestSnapshotId(), logRetryCache); + return prepareFileForWrite(src, myFile, holder, clientMachine, true, + iip.getLatestSnapshotId(), logRetryCache); } catch (IOException ie) { NameNode.stateChangeLog.warn("DIR* NameSystem.append: " +ie.getMessage()); throw ie; @@ -2491,7 +2486,6 @@ private LocatedBlock appendFileInternal(FSPermissionChecker pc, String src, * @param file existing file object * @param leaseHolder identifier of the lease holder on this file * @param clientMachine identifier of the client machine - * @param clientNode if the client is collocated with a DN, that DN's descriptor * @param writeToEditLog whether to persist this change to the edit log * @param logRetryCache whether to record RPC ids in editlog for retry cache * rebuilding @@ -2500,12 +2494,12 @@ private LocatedBlock appendFileInternal(FSPermissionChecker pc, String src, * @throws IOException */ LocatedBlock prepareFileForWrite(String src, INodeFile file, - String leaseHolder, String clientMachine, DatanodeDescriptor clientNode, - boolean writeToEditLog, int latestSnapshot, boolean logRetryCache) + String leaseHolder, String clientMachine, + boolean writeToEditLog, + int latestSnapshot, boolean logRetryCache) throws IOException { file = file.recordModification(latestSnapshot); - final INodeFile cons = file.toUnderConstruction(leaseHolder, clientMachine, - clientNode); + final INodeFile cons = file.toUnderConstruction(leaseHolder, clientMachine); leaseManager.addLease(cons.getFileUnderConstructionFeature() .getClientName(), src); @@ -2777,7 +2771,8 @@ LocatedBlock getAdditionalBlock(String src, long fileId, String clientName, + maxBlocksPerFile); } blockSize = pendingFile.getPreferredBlockSize(); - clientNode = pendingFile.getFileUnderConstructionFeature().getClientNode(); + clientNode = blockManager.getDatanodeManager().getDatanodeByHost( + pendingFile.getFileUnderConstructionFeature().getClientMachine()); replication = pendingFile.getFileReplication(); } finally { readUnlock(); @@ -2983,7 +2978,9 @@ LocatedBlock getAdditionalDatanode(String src, long fileId, if (inode != null) src = inode.getFullPathName(); } final INodeFile file = checkLease(src, clientName, inode, fileId); - clientnode = file.getFileUnderConstructionFeature().getClientNode(); + String clientMachine = file.getFileUnderConstructionFeature() + .getClientMachine(); + clientnode = blockManager.getDatanodeManager().getDatanodeByHost(clientMachine); preferredblocksize = file.getPreferredBlockSize(); //find datanode storages diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java index 782a859e047..896bedb08a5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java @@ -32,15 +32,10 @@ public class FileUnderConstructionFeature implements INode.Feature { private String clientName; // lease holder private final String clientMachine; - // if client is a cluster node too. - private final DatanodeDescriptor clientNode; - public FileUnderConstructionFeature(final String clientName, - final String clientMachine, - final DatanodeDescriptor clientNode) { + public FileUnderConstructionFeature(final String clientName, final String clientMachine) { this.clientName = clientName; this.clientMachine = clientMachine; - this.clientNode = clientNode; } public String getClientName() { @@ -55,10 +50,6 @@ public String getClientMachine() { return clientMachine; } - public DatanodeDescriptor getClientNode() { - return clientNode; - } - /** * Update the length for the last block * diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java index 42025348afc..09ea9c550ff 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java @@ -33,7 +33,6 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction; -import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState; import org.apache.hadoop.hdfs.server.namenode.snapshot.FileDiff; @@ -170,12 +169,11 @@ public boolean isUnderConstruction() { } /** Convert this file to an {@link INodeFileUnderConstruction}. */ - INodeFile toUnderConstruction(String clientName, String clientMachine, - DatanodeDescriptor clientNode) { + INodeFile toUnderConstruction(String clientName, String clientMachine) { Preconditions.checkState(!isUnderConstruction(), "file is already under construction"); FileUnderConstructionFeature uc = new FileUnderConstructionFeature( - clientName, clientMachine, clientNode); + clientName, clientMachine); addFeature(uc); return this; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java index 2bd75f8dd7a..a5e2edf87c9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java @@ -83,7 +83,7 @@ static void addFiles(FSEditLog editLog, int numFiles, short replication, final INodeFile inode = new INodeFile(inodeId.nextValue(), null, p, 0L, 0L, blocks, replication, blockSize); - inode.toUnderConstruction("", "", null); + inode.toUnderConstruction("", ""); // Append path to filename with information about blockIDs String path = "_" + iF + "_B" + blocks[0].getBlockId() + @@ -98,7 +98,7 @@ static void addFiles(FSEditLog editLog, int numFiles, short replication, } INodeFile fileUc = new INodeFile(inodeId.nextValue(), null, p, 0L, 0L, BlockInfo.EMPTY_ARRAY, replication, blockSize); - fileUc.toUnderConstruction("", "", null); + fileUc.toUnderConstruction("", ""); editLog.logOpenFile(filePath, fileUc, false); editLog.logCloseFile(filePath, inode); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java index e4bbf8962da..8074a68e3a9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java @@ -195,7 +195,7 @@ public void run() { for (int i = 0; i < numTransactions; i++) { INodeFile inode = new INodeFile(namesystem.allocateNewInodeId(), null, p, 0L, 0L, BlockInfo.EMPTY_ARRAY, replication, blockSize); - inode.toUnderConstruction("", "", null); + inode.toUnderConstruction("", ""); editLog.logOpenFile("/filename" + (startIndex + i), inode, false); editLog.logCloseFile("/filename" + (startIndex + i), inode); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java index 19b9979502e..cb3ce228c31 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java @@ -29,8 +29,6 @@ import java.util.ArrayList; import java.util.List; -import org.junit.Assert; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -318,7 +316,7 @@ public void testValueOf () throws IOException { {//cast from INodeFileUnderConstruction final INode from = new INodeFile( INodeId.GRANDFATHER_INODE_ID, null, perm, 0L, 0L, null, replication, 1024L); - from.asFile().toUnderConstruction("client", "machine", null); + from.asFile().toUnderConstruction("client", "machine"); //cast to INodeFile, should success final INodeFile f = INodeFile.valueOf(from, path); @@ -1070,12 +1068,11 @@ public void testFileUnderConstruction() { final String clientName = "client"; final String clientMachine = "machine"; - file.toUnderConstruction(clientName, clientMachine, null); + file.toUnderConstruction(clientName, clientMachine); assertTrue(file.isUnderConstruction()); FileUnderConstructionFeature uc = file.getFileUnderConstructionFeature(); assertEquals(clientName, uc.getClientName()); assertEquals(clientMachine, uc.getClientMachine()); - Assert.assertNull(uc.getClientNode()); file.toCompleteFile(Time.now()); assertFalse(file.isUnderConstruction()); From e8ca6480050e38d2fe4859baf4f9a8d22e7f9b85 Mon Sep 17 00:00:00 2001 From: Vinayakumar B Date: Mon, 23 Jun 2014 05:16:05 +0000 Subject: [PATCH 019/112] HDFS-6507. Improve DFSAdmin to support HA cluster better. (Contributd by Zesheng Wu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1604692 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../java/org/apache/hadoop/hdfs/HAUtil.java | 35 +- .../apache/hadoop/hdfs/NameNodeProxies.java | 17 +- .../apache/hadoop/hdfs/tools/DFSAdmin.java | 304 ++++++++++++++---- .../hadoop/hdfs/tools/TestDFSAdminWithHA.java | 231 +++++++++++++ .../src/test/resources/testHDFSConf.xml | 24 +- 6 files changed, 539 insertions(+), 75 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdminWithHA.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index b46aa576cbd..190668ae253 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -461,6 +461,9 @@ Release 2.5.0 - UNRELEASED HDFS-4667. Capture renamed files/directories in snapshot diff report. (jing9 and Binglin Chang via jing9) + HDFS-6507. Improve DFSAdmin to support HA cluster better. + (Zesheng Wu via vinayakumarb) + OPTIMIZATIONS HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java index 568b54489d6..250d41c5cba 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java @@ -39,6 +39,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.NameNodeProxies; +import org.apache.hadoop.hdfs.NameNodeProxies.ProxyAndInfo; import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; @@ -353,18 +354,42 @@ public static InetSocketAddress getAddressOfActive(FileSystem fs) */ public static List getProxiesForAllNameNodesInNameservice( Configuration conf, String nsId) throws IOException { + List> proxies = + getProxiesForAllNameNodesInNameservice(conf, nsId, ClientProtocol.class); + + List namenodes = new ArrayList( + proxies.size()); + for (ProxyAndInfo proxy : proxies) { + namenodes.add(proxy.getProxy()); + } + return namenodes; + } + + /** + * Get an RPC proxy for each NN in an HA nameservice. Used when a given RPC + * call should be made on every NN in an HA nameservice, not just the active. + * + * @param conf configuration + * @param nsId the nameservice to get all of the proxies for. + * @param xface the protocol class. + * @return a list of RPC proxies for each NN in the nameservice. + * @throws IOException in the event of error. + */ + public static List> getProxiesForAllNameNodesInNameservice( + Configuration conf, String nsId, Class xface) throws IOException { Map nnAddresses = DFSUtil.getRpcAddressesForNameserviceId(conf, nsId, null); - List namenodes = new ArrayList(); + List> proxies = new ArrayList>( + nnAddresses.size()); for (InetSocketAddress nnAddress : nnAddresses.values()) { - NameNodeProxies.ProxyAndInfo proxyInfo = null; + NameNodeProxies.ProxyAndInfo proxyInfo = null; proxyInfo = NameNodeProxies.createNonHAProxy(conf, - nnAddress, ClientProtocol.class, + nnAddress, xface, UserGroupInformation.getCurrentUser(), false); - namenodes.add(proxyInfo.getProxy()); + proxies.add(proxyInfo); } - return namenodes; + return proxies; } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java index 6c82c1837a0..2bcb2a16222 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/NameNodeProxies.java @@ -106,10 +106,13 @@ public class NameNodeProxies { public static class ProxyAndInfo { private final PROXYTYPE proxy; private final Text dtService; + private final InetSocketAddress address; - public ProxyAndInfo(PROXYTYPE proxy, Text dtService) { + public ProxyAndInfo(PROXYTYPE proxy, Text dtService, + InetSocketAddress address) { this.proxy = proxy; this.dtService = dtService; + this.address = address; } public PROXYTYPE getProxy() { @@ -119,6 +122,10 @@ public PROXYTYPE getProxy() { public Text getDelegationTokenService() { return dtService; } + + public InetSocketAddress getAddress() { + return address; + } } /** @@ -161,7 +168,8 @@ public static ProxyAndInfo createProxy(Configuration conf, dtService = SecurityUtil.buildTokenService( NameNode.getAddress(nameNodeUri)); } - return new ProxyAndInfo(proxy, dtService); + return new ProxyAndInfo(proxy, dtService, + NameNode.getAddress(nameNodeUri)); } } @@ -221,7 +229,8 @@ public static ProxyAndInfo createProxyWithLossyRetryHandler( dtService = SecurityUtil.buildTokenService( NameNode.getAddress(nameNodeUri)); } - return new ProxyAndInfo(proxy, dtService); + return new ProxyAndInfo(proxy, dtService, + NameNode.getAddress(nameNodeUri)); } else { LOG.warn("Currently creating proxy using " + "LossyRetryInvocationHandler requires NN HA setup"); @@ -274,7 +283,7 @@ public static ProxyAndInfo createNonHAProxy( throw new IllegalStateException(message); } - return new ProxyAndInfo(proxy, dtService); + return new ProxyAndInfo(proxy, dtService, nnAddr); } private static JournalProtocol createNNProxyWithJournalProtocol( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java index e3cdd639c96..2c1bf038cc4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java @@ -49,6 +49,7 @@ import org.apache.hadoop.hdfs.HAUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.NameNodeProxies; +import org.apache.hadoop.hdfs.NameNodeProxies.ProxyAndInfo; import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol; import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; @@ -498,25 +499,60 @@ public void setSafeMode(String[] argv, int idx) throws IOException { printUsage("-safemode"); return; } - DistributedFileSystem dfs = getDFS(); - boolean inSafeMode = dfs.setSafeMode(action); - // - // If we are waiting for safemode to exit, then poll and - // sleep till we are out of safemode. - // - if (waitExitSafe) { - while (inSafeMode) { - try { - Thread.sleep(5000); - } catch (java.lang.InterruptedException e) { - throw new IOException("Wait Interrupted"); + DistributedFileSystem dfs = getDFS(); + Configuration dfsConf = dfs.getConf(); + URI dfsUri = dfs.getUri(); + boolean isHaEnabled = HAUtil.isLogicalUri(dfsConf, dfsUri); + + if (isHaEnabled) { + String nsId = dfsUri.getHost(); + List> proxies = + HAUtil.getProxiesForAllNameNodesInNameservice( + dfsConf, nsId, ClientProtocol.class); + for (ProxyAndInfo proxy : proxies) { + ClientProtocol haNn = proxy.getProxy(); + boolean inSafeMode = haNn.setSafeMode(action, false); + if (waitExitSafe) { + inSafeMode = waitExitSafeMode(haNn, inSafeMode); } - inSafeMode = dfs.setSafeMode(SafeModeAction.SAFEMODE_GET); + System.out.println("Safe mode is " + (inSafeMode ? "ON" : "OFF") + + " in " + proxy.getAddress()); } + } else { + boolean inSafeMode = dfs.setSafeMode(action); + if (waitExitSafe) { + inSafeMode = waitExitSafeMode(dfs, inSafeMode); + } + System.out.println("Safe mode is " + (inSafeMode ? "ON" : "OFF")); } - System.out.println("Safe mode is " + (inSafeMode ? "ON" : "OFF")); + } + + private boolean waitExitSafeMode(DistributedFileSystem dfs, boolean inSafeMode) + throws IOException { + while (inSafeMode) { + try { + Thread.sleep(5000); + } catch (java.lang.InterruptedException e) { + throw new IOException("Wait Interrupted"); + } + inSafeMode = dfs.setSafeMode(SafeModeAction.SAFEMODE_GET, false); + } + return inSafeMode; + } + + private boolean waitExitSafeMode(ClientProtocol nn, boolean inSafeMode) + throws IOException { + while (inSafeMode) { + try { + Thread.sleep(5000); + } catch (java.lang.InterruptedException e) { + throw new IOException("Wait Interrupted"); + } + inSafeMode = nn.setSafeMode(SafeModeAction.SAFEMODE_GET, false); + } + return inSafeMode; } /** @@ -561,7 +597,24 @@ public int saveNamespace() throws IOException { int exitCode = -1; DistributedFileSystem dfs = getDFS(); - dfs.saveNamespace(); + Configuration dfsConf = dfs.getConf(); + URI dfsUri = dfs.getUri(); + boolean isHaEnabled = HAUtil.isLogicalUri(dfsConf, dfsUri); + + if (isHaEnabled) { + String nsId = dfsUri.getHost(); + List> proxies = + HAUtil.getProxiesForAllNameNodesInNameservice(dfsConf, + nsId, ClientProtocol.class); + for (ProxyAndInfo proxy : proxies) { + proxy.getProxy().saveNamespace(); + System.out.println("Save namespace successful for " + + proxy.getAddress()); + } + } else { + dfs.saveNamespace(); + System.out.println("Save namespace successful"); + } exitCode = 0; return exitCode; @@ -583,15 +636,30 @@ public int rollEdits() throws IOException { */ public int restoreFailedStorage(String arg) throws IOException { int exitCode = -1; - if(!arg.equals("check") && !arg.equals("true") && !arg.equals("false")) { System.err.println("restoreFailedStorage valid args are true|false|check"); return exitCode; } DistributedFileSystem dfs = getDFS(); - Boolean res = dfs.restoreFailedStorage(arg); - System.out.println("restoreFailedStorage is set to " + res); + Configuration dfsConf = dfs.getConf(); + URI dfsUri = dfs.getUri(); + boolean isHaEnabled = HAUtil.isLogicalUri(dfsConf, dfsUri); + + if (isHaEnabled) { + String nsId = dfsUri.getHost(); + List> proxies = + HAUtil.getProxiesForAllNameNodesInNameservice(dfsConf, + nsId, ClientProtocol.class); + for (ProxyAndInfo proxy : proxies) { + Boolean res = proxy.getProxy().restoreFailedStorage(arg); + System.out.println("restoreFailedStorage is set to " + res + " for " + + proxy.getAddress()); + } + } else { + Boolean res = dfs.restoreFailedStorage(arg); + System.out.println("restoreFailedStorage is set to " + res); + } exitCode = 0; return exitCode; @@ -607,7 +675,24 @@ public int refreshNodes() throws IOException { int exitCode = -1; DistributedFileSystem dfs = getDFS(); - dfs.refreshNodes(); + Configuration dfsConf = dfs.getConf(); + URI dfsUri = dfs.getUri(); + boolean isHaEnabled = HAUtil.isLogicalUri(dfsConf, dfsUri); + + if (isHaEnabled) { + String nsId = dfsUri.getHost(); + List> proxies = + HAUtil.getProxiesForAllNameNodesInNameservice(dfsConf, + nsId, ClientProtocol.class); + for (ProxyAndInfo proxy: proxies) { + proxy.getProxy().refreshNodes(); + System.out.println("Refresh nodes successful for " + + proxy.getAddress()); + } + } else { + dfs.refreshNodes(); + System.out.println("Refresh nodes successful"); + } exitCode = 0; return exitCode; @@ -641,7 +726,24 @@ public int setBalancerBandwidth(String[] argv, int idx) throws IOException { } DistributedFileSystem dfs = (DistributedFileSystem) fs; - dfs.setBalancerBandwidth(bandwidth); + Configuration dfsConf = dfs.getConf(); + URI dfsUri = dfs.getUri(); + boolean isHaEnabled = HAUtil.isLogicalUri(dfsConf, dfsUri); + + if (isHaEnabled) { + String nsId = dfsUri.getHost(); + List> proxies = + HAUtil.getProxiesForAllNameNodesInNameservice(dfsConf, + nsId, ClientProtocol.class); + for (ProxyAndInfo proxy : proxies) { + proxy.getProxy().setBalancerBandwidth(bandwidth); + System.out.println("Balancer bandwidth is set to " + bandwidth + + " for " + proxy.getAddress()); + } + } else { + dfs.setBalancerBandwidth(bandwidth); + System.out.println("Balancer bandwidth is set to " + bandwidth); + } exitCode = 0; return exitCode; @@ -937,11 +1039,18 @@ public int finalizeUpgrade() throws IOException { if (!HAUtil.isAtLeastOneActive(namenodes)) { throw new IOException("Cannot finalize with no NameNode active"); } - for (ClientProtocol haNn : namenodes) { - haNn.finalizeUpgrade(); + + List> proxies = + HAUtil.getProxiesForAllNameNodesInNameservice(dfsConf, + nsId, ClientProtocol.class); + for (ProxyAndInfo proxy : proxies) { + proxy.getProxy().finalizeUpgrade(); + System.out.println("Finalize upgrade successful for " + + proxy.getAddress()); } } else { dfs.finalizeUpgrade(); + System.out.println("Finalize upgrade successful"); } return 0; @@ -958,9 +1067,25 @@ public int finalizeUpgrade() throws IOException { public int metaSave(String[] argv, int idx) throws IOException { String pathname = argv[idx]; DistributedFileSystem dfs = getDFS(); - dfs.metaSave(pathname); - System.out.println("Created metasave file " + pathname + " in the log " + - "directory of namenode " + dfs.getUri()); + Configuration dfsConf = dfs.getConf(); + URI dfsUri = dfs.getUri(); + boolean isHaEnabled = HAUtil.isLogicalUri(dfsConf, dfsUri); + + if (isHaEnabled) { + String nsId = dfsUri.getHost(); + List> proxies = + HAUtil.getProxiesForAllNameNodesInNameservice(dfsConf, + nsId, ClientProtocol.class); + for (ProxyAndInfo proxy : proxies) { + proxy.getProxy().metaSave(pathname); + System.out.println("Created metasave file " + pathname + " in the log " + + "directory of namenode " + proxy.getAddress()); + } + } else { + dfs.metaSave(pathname); + System.out.println("Created metasave file " + pathname + " in the log " + + "directory of namenode " + dfs.getUri()); + } return 0; } @@ -1022,20 +1147,37 @@ private static UserGroupInformation getUGI() public int refreshServiceAcl() throws IOException { // Get the current configuration Configuration conf = getConf(); - + // for security authorization // server principal for this call // should be NN's one. conf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY, conf.get(DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY, "")); - // Create the client - RefreshAuthorizationPolicyProtocol refreshProtocol = - NameNodeProxies.createProxy(conf, FileSystem.getDefaultUri(conf), - RefreshAuthorizationPolicyProtocol.class).getProxy(); - - // Refresh the authorization policy in-effect - refreshProtocol.refreshServiceAcl(); + DistributedFileSystem dfs = getDFS(); + URI dfsUri = dfs.getUri(); + boolean isHaEnabled = HAUtil.isLogicalUri(conf, dfsUri); + + if (isHaEnabled) { + // Run refreshServiceAcl for all NNs if HA is enabled + String nsId = dfsUri.getHost(); + List> proxies = + HAUtil.getProxiesForAllNameNodesInNameservice(conf, nsId, + RefreshAuthorizationPolicyProtocol.class); + for (ProxyAndInfo proxy : proxies) { + proxy.getProxy().refreshServiceAcl(); + System.out.println("Refresh service acl successful for " + + proxy.getAddress()); + } + } else { + // Create the client + RefreshAuthorizationPolicyProtocol refreshProtocol = + NameNodeProxies.createProxy(conf, FileSystem.getDefaultUri(conf), + RefreshAuthorizationPolicyProtocol.class).getProxy(); + // Refresh the authorization policy in-effect + refreshProtocol.refreshServiceAcl(); + System.out.println("Refresh service acl successful"); + } return 0; } @@ -1054,14 +1196,32 @@ public int refreshUserToGroupsMappings() throws IOException { // should be NN's one. conf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY, conf.get(DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY, "")); - - // Create the client - RefreshUserMappingsProtocol refreshProtocol = - NameNodeProxies.createProxy(conf, FileSystem.getDefaultUri(conf), - RefreshUserMappingsProtocol.class).getProxy(); - // Refresh the user-to-groups mappings - refreshProtocol.refreshUserToGroupsMappings(); + DistributedFileSystem dfs = getDFS(); + URI dfsUri = dfs.getUri(); + boolean isHaEnabled = HAUtil.isLogicalUri(conf, dfsUri); + + if (isHaEnabled) { + // Run refreshUserToGroupsMapings for all NNs if HA is enabled + String nsId = dfsUri.getHost(); + List> proxies = + HAUtil.getProxiesForAllNameNodesInNameservice(conf, nsId, + RefreshUserMappingsProtocol.class); + for (ProxyAndInfo proxy : proxies) { + proxy.getProxy().refreshUserToGroupsMappings(); + System.out.println("Refresh user to groups mapping successful for " + + proxy.getAddress()); + } + } else { + // Create the client + RefreshUserMappingsProtocol refreshProtocol = + NameNodeProxies.createProxy(conf, FileSystem.getDefaultUri(conf), + RefreshUserMappingsProtocol.class).getProxy(); + + // Refresh the user-to-groups mappings + refreshProtocol.refreshUserToGroupsMappings(); + System.out.println("Refresh user to groups mapping successful"); + } return 0; } @@ -1082,13 +1242,31 @@ public int refreshSuperUserGroupsConfiguration() throws IOException { conf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY, conf.get(DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY, "")); - // Create the client - RefreshUserMappingsProtocol refreshProtocol = - NameNodeProxies.createProxy(conf, FileSystem.getDefaultUri(conf), - RefreshUserMappingsProtocol.class).getProxy(); + DistributedFileSystem dfs = getDFS(); + URI dfsUri = dfs.getUri(); + boolean isHaEnabled = HAUtil.isLogicalUri(conf, dfsUri); - // Refresh the user-to-groups mappings - refreshProtocol.refreshSuperUserGroupsConfiguration(); + if (isHaEnabled) { + // Run refreshSuperUserGroupsConfiguration for all NNs if HA is enabled + String nsId = dfsUri.getHost(); + List> proxies = + HAUtil.getProxiesForAllNameNodesInNameservice(conf, nsId, + RefreshUserMappingsProtocol.class); + for (ProxyAndInfo proxy : proxies) { + proxy.getProxy().refreshSuperUserGroupsConfiguration(); + System.out.println("Refresh super user groups configuration " + + "successful for " + proxy.getAddress()); + } + } else { + // Create the client + RefreshUserMappingsProtocol refreshProtocol = + NameNodeProxies.createProxy(conf, FileSystem.getDefaultUri(conf), + RefreshUserMappingsProtocol.class).getProxy(); + + // Refresh the user-to-groups mappings + refreshProtocol.refreshSuperUserGroupsConfiguration(); + System.out.println("Refresh super user groups configuration successful"); + } return 0; } @@ -1102,15 +1280,33 @@ public int refreshCallQueue() throws IOException { // should be NN's one. conf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY, conf.get(DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY, "")); - - // Create the client - RefreshCallQueueProtocol refreshProtocol = - NameNodeProxies.createProxy(conf, FileSystem.getDefaultUri(conf), - RefreshCallQueueProtocol.class).getProxy(); - // Refresh the call queue - refreshProtocol.refreshCallQueue(); - + DistributedFileSystem dfs = getDFS(); + URI dfsUri = dfs.getUri(); + boolean isHaEnabled = HAUtil.isLogicalUri(conf, dfsUri); + + if (isHaEnabled) { + // Run refreshCallQueue for all NNs if HA is enabled + String nsId = dfsUri.getHost(); + List> proxies = + HAUtil.getProxiesForAllNameNodesInNameservice(conf, nsId, + RefreshCallQueueProtocol.class); + for (ProxyAndInfo proxy : proxies) { + proxy.getProxy().refreshCallQueue(); + System.out.println("Refresh call queue successful for " + + proxy.getAddress()); + } + } else { + // Create the client + RefreshCallQueueProtocol refreshProtocol = + NameNodeProxies.createProxy(conf, FileSystem.getDefaultUri(conf), + RefreshCallQueueProtocol.class).getProxy(); + + // Refresh the call queue + refreshProtocol.refreshCallQueue(); + System.out.println("Refresh call queue successful"); + } + return 0; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdminWithHA.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdminWithHA.java new file mode 100644 index 00000000000..40826134d96 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdminWithHA.java @@ -0,0 +1,231 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.tools; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; + +import com.google.common.base.Charsets; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSUtil; +import org.apache.hadoop.hdfs.HAUtil; +import org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster; +import org.junit.After; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class TestDFSAdminWithHA { + + private final ByteArrayOutputStream out = new ByteArrayOutputStream(); + private final ByteArrayOutputStream err = new ByteArrayOutputStream(); + private MiniQJMHACluster cluster; + private Configuration conf; + private DFSAdmin admin; + private PrintStream originOut; + private PrintStream originErr; + + private static final String NSID = "ns1"; + + private void assertOutputMatches(String string) { + String errOutput = new String(out.toByteArray(), Charsets.UTF_8); + String output = new String(out.toByteArray(), Charsets.UTF_8); + + if (!errOutput.matches(string) && !output.matches(string)) { + fail("Expected output to match '" + string + + "' but err_output was:\n" + errOutput + + "\n and output was: \n" + output); + } + + out.reset(); + err.reset(); + } + + private void setHAConf(Configuration conf, String nn1Addr, String nn2Addr) { + conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, + "hdfs://" + NSID); + conf.set(DFSConfigKeys.DFS_NAMESERVICES, NSID); + conf.set(DFSConfigKeys.DFS_NAMESERVICE_ID, NSID); + conf.set(DFSUtil.addKeySuffixes( + DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX, NSID), "nn1,nn2"); + conf.set(DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY, "nn1"); + conf.set(DFSUtil.addKeySuffixes( + DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY, NSID, "nn1"), nn1Addr); + conf.set(DFSUtil.addKeySuffixes( + DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY, NSID, "nn2"), nn2Addr); + } + + private void setUpHaCluster(boolean security) throws Exception { + conf = new Configuration(); + conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, + security); + cluster = new MiniQJMHACluster.Builder(conf).build(); + setHAConf(conf, cluster.getDfsCluster().getNameNode(0).getHostAndPort(), + cluster.getDfsCluster().getNameNode(1).getHostAndPort()); + cluster.getDfsCluster().getNameNode(0).getHostAndPort(); + admin = new DFSAdmin(); + admin.setConf(conf); + assertTrue(HAUtil.isHAEnabled(conf, "ns1")); + + originOut = System.out; + originErr = System.err; + System.setOut(new PrintStream(out)); + System.setErr(new PrintStream(err)); + } + + @After + public void tearDown() throws Exception { + System.out.flush(); + System.err.flush(); + System.setOut(originOut); + System.setErr(originErr); + } + + @Test(timeout = 30000) + public void testSetSafeMode() throws Exception { + setUpHaCluster(false); + // Enter safemode + int exitCode = admin.run(new String[] {"-safemode", "enter"}); + assertEquals(err.toString().trim(), 0, exitCode); + String message = "Safe mode is ON in.*"; + assertOutputMatches(message + "\n" + message + "\n"); + + // Get safemode + exitCode = admin.run(new String[] {"-safemode", "get"}); + assertEquals(err.toString().trim(), 0, exitCode); + message = "Safe mode is ON in.*"; + assertOutputMatches(message + "\n" + message + "\n"); + + // Leave safemode + exitCode = admin.run(new String[] {"-safemode", "leave"}); + assertEquals(err.toString().trim(), 0, exitCode); + message = "Safe mode is OFF in.*"; + assertOutputMatches(message + "\n" + message + "\n"); + + // Get safemode + exitCode = admin.run(new String[] {"-safemode", "get"}); + assertEquals(err.toString().trim(), 0, exitCode); + message = "Safe mode is OFF in.*"; + assertOutputMatches(message + "\n" + message + "\n"); + } + + @Test (timeout = 30000) + public void testSaveNamespace() throws Exception { + setUpHaCluster(false); + // Safe mode should be turned ON in order to create namespace image. + int exitCode = admin.run(new String[] {"-safemode", "enter"}); + assertEquals(err.toString().trim(), 0, exitCode); + String message = "Safe mode is ON in.*"; + assertOutputMatches(message + "\n" + message + "\n"); + + exitCode = admin.run(new String[] {"-saveNamespace"}); + assertEquals(err.toString().trim(), 0, exitCode); + message = "Save namespace successful for.*"; + assertOutputMatches(message + "\n" + message + "\n"); + } + + @Test (timeout = 30000) + public void testRestoreFailedStorage() throws Exception { + setUpHaCluster(false); + int exitCode = admin.run(new String[] {"-restoreFailedStorage", "check"}); + assertEquals(err.toString().trim(), 0, exitCode); + String message = "restoreFailedStorage is set to false for.*"; + // Default is false + assertOutputMatches(message + "\n" + message + "\n"); + + exitCode = admin.run(new String[] {"-restoreFailedStorage", "true"}); + assertEquals(err.toString().trim(), 0, exitCode); + message = "restoreFailedStorage is set to true for.*"; + assertOutputMatches(message + "\n" + message + "\n"); + + exitCode = admin.run(new String[] {"-restoreFailedStorage", "false"}); + assertEquals(err.toString().trim(), 0, exitCode); + message = "restoreFailedStorage is set to false for.*"; + assertOutputMatches(message + "\n" + message + "\n"); + } + + @Test (timeout = 30000) + public void testRefreshNodes() throws Exception { + setUpHaCluster(false); + int exitCode = admin.run(new String[] {"-refreshNodes"}); + assertEquals(err.toString().trim(), 0, exitCode); + String message = "Refresh nodes successful for.*"; + assertOutputMatches(message + "\n" + message + "\n"); + } + + @Test (timeout = 30000) + public void testSetBalancerBandwidth() throws Exception { + setUpHaCluster(false); + int exitCode = admin.run(new String[] {"-setBalancerBandwidth", "10"}); + assertEquals(err.toString().trim(), 0, exitCode); + String message = "Balancer bandwidth is set to 10 for.*"; + assertOutputMatches(message + "\n" + message + "\n"); + } + + @Test (timeout = 30000) + public void testMetaSave() throws Exception { + setUpHaCluster(false); + int exitCode = admin.run(new String[] {"-metasave", "dfs.meta"}); + assertEquals(err.toString().trim(), 0, exitCode); + String message = "Created metasave file dfs.meta in the log directory" + + " of namenode.*"; + assertOutputMatches(message + "\n" + message + "\n"); + } + + @Test (timeout = 30000) + public void testRefreshServiceAcl() throws Exception { + setUpHaCluster(true); + int exitCode = admin.run(new String[] {"-refreshServiceAcl"}); + assertEquals(err.toString().trim(), 0, exitCode); + String message = "Refresh service acl successful for.*"; + assertOutputMatches(message + "\n" + message + "\n"); + } + + @Test (timeout = 30000) + public void testRefreshUserToGroupsMappings() throws Exception { + setUpHaCluster(false); + int exitCode = admin.run(new String[] {"-refreshUserToGroupsMappings"}); + assertEquals(err.toString().trim(), 0, exitCode); + String message = "Refresh user to groups mapping successful for.*"; + assertOutputMatches(message + "\n" + message + "\n"); + } + + @Test (timeout = 30000) + public void testRefreshSuperUserGroupsConfiguration() throws Exception { + setUpHaCluster(false); + int exitCode = admin.run( + new String[] {"-refreshSuperUserGroupsConfiguration"}); + assertEquals(err.toString().trim(), 0, exitCode); + String message = "Refresh super user groups configuration successful for.*"; + assertOutputMatches(message + "\n" + message + "\n"); + } + + @Test (timeout = 30000) + public void testRefreshCallQueue() throws Exception { + setUpHaCluster(false); + int exitCode = admin.run(new String[] {"-refreshCallQueue"}); + assertEquals(err.toString().trim(), 0, exitCode); + String message = "Refresh call queue successful for.*"; + assertOutputMatches(message + "\n" + message + "\n"); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml index 2ec4fcf3757..03083c50cea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml @@ -15714,8 +15714,8 @@ - ExactComparator - + RegexpComparator + Refresh service acl successful(\n)* + + + +

    +Infrastructure for a Metrics2 source that provides information on Windows +Azure Filesystem for Hadoop instances. +

    + + + diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java index 81339541bdd..02738e7efb5 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java @@ -27,9 +27,18 @@ import java.util.EnumSet; import java.util.GregorianCalendar; import java.util.TimeZone; +import java.util.concurrent.ConcurrentLinkedQueue; +import org.apache.commons.configuration.SubsetConfiguration; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation; +import org.apache.hadoop.fs.azure.metrics.AzureFileSystemMetricsSystem; +import org.apache.hadoop.metrics2.AbstractMetric; +import org.apache.hadoop.metrics2.MetricsRecord; +import org.apache.hadoop.metrics2.MetricsSink; +import org.apache.hadoop.metrics2.MetricsTag; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import com.microsoft.windowsazure.storage.AccessCondition; import com.microsoft.windowsazure.storage.CloudStorageAccount; @@ -76,7 +85,10 @@ public final class AzureBlobStorageTestAccount { private NativeAzureFileSystem fs; private AzureNativeFileSystemStore storage; private MockStorageInterface mockStorage; - + private static final ConcurrentLinkedQueue allMetrics = + new ConcurrentLinkedQueue(); + + private AzureBlobStorageTestAccount(NativeAzureFileSystem fs, CloudStorageAccount account, CloudBlobContainer container) { this.account = account; @@ -124,6 +136,10 @@ private AzureBlobStorageTestAccount(NativeAzureFileSystem fs, this.fs = fs; this.mockStorage = mockStorage; } + + private static void addRecord(MetricsRecord record) { + allMetrics.add(record); + } public static String getMockContainerUri() { return String.format("http://%s/%s", @@ -141,6 +157,47 @@ public static String toMockUri(Path path) { // Remove the first SEPARATOR return toMockUri(path.toUri().getRawPath().substring(1)); } + + public Number getLatestMetricValue(String metricName, Number defaultValue) + throws IndexOutOfBoundsException{ + boolean found = false; + Number ret = null; + for (MetricsRecord currentRecord : allMetrics) { + // First check if this record is coming for my file system. + if (wasGeneratedByMe(currentRecord)) { + for (AbstractMetric currentMetric : currentRecord.metrics()) { + if (currentMetric.name().equalsIgnoreCase(metricName)) { + found = true; + ret = currentMetric.value(); + break; + } + } + } + } + if (!found) { + if (defaultValue != null) { + return defaultValue; + } + throw new IndexOutOfBoundsException(metricName); + } + return ret; + } + + /** + * Checks if the given record was generated by my WASB file system instance. + * @param currentRecord The metrics record to check. + * @return + */ + private boolean wasGeneratedByMe(MetricsRecord currentRecord) { + String myFsId = fs.getInstrumentation().getFileSystemInstanceId().toString(); + for (MetricsTag currentTag : currentRecord.tags()) { + if (currentTag.name().equalsIgnoreCase("wasbFileSystemId")) { + return currentTag.value().equals(myFsId); + } + } + return false; + } + /** * Gets the blob reference to the given blob key. @@ -236,7 +293,6 @@ public static AzureBlobStorageTestAccount createForEmulator() public static AzureBlobStorageTestAccount createOutOfBandStore( int uploadBlockSize, int downloadBlockSize) throws Exception { - CloudBlobContainer container = null; Configuration conf = createTestConfiguration(); CloudStorageAccount account = createTestAccount(conf); @@ -262,11 +318,25 @@ public static AzureBlobStorageTestAccount createOutOfBandStore( // Set account URI and initialize Azure file system. URI accountUri = createAccountUri(accountName, containerName); + // Set up instrumentation. + // + AzureFileSystemMetricsSystem.fileSystemStarted(); + String sourceName = NativeAzureFileSystem.newMetricsSourceName(); + String sourceDesc = "Azure Storage Volume File System metrics"; + + AzureFileSystemInstrumentation instrumentation = + DefaultMetricsSystem.instance().register(sourceName, + sourceDesc, new AzureFileSystemInstrumentation(conf)); + + AzureFileSystemMetricsSystem.registerSource( + sourceName, sourceDesc, instrumentation); + + // Create a new AzureNativeFileSystemStore object. AzureNativeFileSystemStore testStorage = new AzureNativeFileSystemStore(); // Initialize the store with the throttling feedback interfaces. - testStorage.initialize(accountUri, conf); + testStorage.initialize(accountUri, conf, instrumentation); // Create test account initializing the appropriate member variables. AzureBlobStorageTestAccount testAcct = new AzureBlobStorageTestAccount( @@ -722,5 +792,20 @@ public CloudStorageAccount getRealAccount() { public MockStorageInterface getMockStorage() { return mockStorage; } + + public static class StandardCollector implements MetricsSink { + @Override + public void init(SubsetConfiguration conf) { + } + + @Override + public void putMetrics(MetricsRecord record) { + addRecord(record); + } + + @Override + public void flush() { + } + } } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java index 88d976ce8e9..6e89822028c 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestAzureFileSystemErrorConditions.java @@ -33,11 +33,9 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.TestHookOperationContext; -import org.apache.hadoop.fs.permission.FsPermission; import org.junit.Test; import com.microsoft.windowsazure.storage.OperationContext; diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/AzureMetricsTestUtil.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/AzureMetricsTestUtil.java new file mode 100644 index 00000000000..12694173c3a --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/AzureMetricsTestUtil.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure.metrics; + +import static org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation.WASB_BYTES_READ; +import static org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation.WASB_BYTES_WRITTEN; +import static org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation.WASB_RAW_BYTES_DOWNLOADED; +import static org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation.WASB_RAW_BYTES_UPLOADED; +import static org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation.WASB_WEB_RESPONSES; +import static org.apache.hadoop.test.MetricsAsserts.getLongCounter; +import static org.apache.hadoop.test.MetricsAsserts.getLongGauge; +import static org.apache.hadoop.test.MetricsAsserts.getMetrics; + +public final class AzureMetricsTestUtil { + public static long getLongGaugeValue(AzureFileSystemInstrumentation instrumentation, + String gaugeName) { + return getLongGauge(gaugeName, getMetrics(instrumentation)); + } + + /** + * Gets the current value of the given counter. + */ + public static long getLongCounterValue(AzureFileSystemInstrumentation instrumentation, + String counterName) { + return getLongCounter(counterName, getMetrics(instrumentation)); + } + + + /** + * Gets the current value of the wasb_bytes_written_last_second counter. + */ + public static long getCurrentBytesWritten(AzureFileSystemInstrumentation instrumentation) { + return getLongGaugeValue(instrumentation, WASB_BYTES_WRITTEN); + } + + /** + * Gets the current value of the wasb_bytes_read_last_second counter. + */ + public static long getCurrentBytesRead(AzureFileSystemInstrumentation instrumentation) { + return getLongGaugeValue(instrumentation, WASB_BYTES_READ); + } + + /** + * Gets the current value of the wasb_raw_bytes_uploaded counter. + */ + public static long getCurrentTotalBytesWritten( + AzureFileSystemInstrumentation instrumentation) { + return getLongCounterValue(instrumentation, WASB_RAW_BYTES_UPLOADED); + } + + /** + * Gets the current value of the wasb_raw_bytes_downloaded counter. + */ + public static long getCurrentTotalBytesRead( + AzureFileSystemInstrumentation instrumentation) { + return getLongCounterValue(instrumentation, WASB_RAW_BYTES_DOWNLOADED); + } + + /** + * Gets the current value of the asv_web_responses counter. + */ + public static long getCurrentWebResponses( + AzureFileSystemInstrumentation instrumentation) { + return getLongCounter(WASB_WEB_RESPONSES, getMetrics(instrumentation)); + } +} diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestAzureFileSystemInstrumentation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestAzureFileSystemInstrumentation.java new file mode 100644 index 00000000000..35004d60d7a --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestAzureFileSystemInstrumentation.java @@ -0,0 +1,546 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure.metrics; + +import static org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation.WASB_CLIENT_ERRORS; +import static org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation.WASB_DIRECTORIES_CREATED; +import static org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation.WASB_DOWNLOAD_LATENCY; +import static org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation.WASB_DOWNLOAD_RATE; +import static org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation.WASB_FILES_CREATED; +import static org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation.WASB_FILES_DELETED; +import static org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation.WASB_SERVER_ERRORS; +import static org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation.WASB_UPLOAD_LATENCY; +import static org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation.WASB_UPLOAD_RATE; +import static org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation.WASB_WEB_RESPONSES; +import static org.apache.hadoop.test.MetricsAsserts.assertCounter; +import static org.apache.hadoop.test.MetricsAsserts.getMetrics; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeNotNull; +import static org.mockito.Matchers.argThat; +import static org.mockito.Mockito.verify; + +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.Date; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount; +import org.apache.hadoop.fs.azure.AzureException; +import org.apache.hadoop.fs.azure.AzureNativeFileSystemStore; +import org.apache.hadoop.fs.azure.NativeAzureFileSystem; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; +import org.apache.hadoop.metrics2.MetricsTag; +import org.hamcrest.BaseMatcher; +import org.hamcrest.Description; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class TestAzureFileSystemInstrumentation { + private FileSystem fs; + private AzureBlobStorageTestAccount testAccount; + + @Before + public void setUp() throws Exception { + testAccount = AzureBlobStorageTestAccount.create(); + if (testAccount != null) { + fs = testAccount.getFileSystem(); + } + assumeNotNull(testAccount); + } + + @After + public void tearDown() throws Exception { + if (testAccount != null) { + testAccount.cleanup(); + testAccount = null; + fs = null; + } + } + + @Test + public void testMetricTags() throws Exception { + String accountName = + testAccount.getRealAccount().getBlobEndpoint() + .getAuthority(); + String containerName = + testAccount.getRealContainer().getName(); + MetricsRecordBuilder myMetrics = getMyMetrics(); + verify(myMetrics).add(argThat( + new TagMatcher("accountName", accountName) + )); + verify(myMetrics).add(argThat( + new TagMatcher("containerName", containerName) + )); + verify(myMetrics).add(argThat( + new TagMatcher("Context", "azureFileSystem") + )); + verify(myMetrics).add(argThat( + new TagExistsMatcher("wasbFileSystemId") + )); + } + + + @Test + public void testMetricsOnMkdirList() throws Exception { + long base = getBaseWebResponses(); + + // Create a directory + assertTrue(fs.mkdirs(new Path("a"))); + // At the time of writing, it takes 1 request to create the actual directory, + // plus 2 requests per level to check that there's no blob with that name and + // 1 request per level above to create it if it doesn't exist. + // So for the path above (/user//a), it takes 2 requests each to check + // there's no blob called /user, no blob called /user/ and no blob + // called /user//a, and then 3 request for the creation of the three + // levels, and then 2 requests for checking/stamping the version of AS, + // totaling 11. + // Also, there's the initial 1 request for container check so total is 12. + base = assertWebResponsesInRange(base, 1, 12); + assertEquals(1, + AzureMetricsTestUtil.getLongCounterValue(getInstrumentation(), WASB_DIRECTORIES_CREATED)); + + // List the root contents + assertEquals(1, fs.listStatus(new Path("/")).length); + base = assertWebResponsesEquals(base, 1); + + assertNoErrors(); + } + + private BandwidthGaugeUpdater getBandwidthGaugeUpdater() { + NativeAzureFileSystem azureFs = (NativeAzureFileSystem)fs; + AzureNativeFileSystemStore azureStore = azureFs.getStore(); + return azureStore.getBandwidthGaugeUpdater(); + } + + private static byte[] nonZeroByteArray(int size) { + byte[] data = new byte[size]; + Arrays.fill(data, (byte)5); + return data; + } + + @Test + public void testMetricsOnFileCreateRead() throws Exception { + long base = getBaseWebResponses(); + + assertEquals(0, AzureMetricsTestUtil.getCurrentBytesWritten(getInstrumentation())); + + Path filePath = new Path("/metricsTest_webResponses"); + final int FILE_SIZE = 1000; + + // Suppress auto-update of bandwidth metrics so we get + // to update them exactly when we want to. + getBandwidthGaugeUpdater().suppressAutoUpdate(); + + // Create a file + Date start = new Date(); + OutputStream outputStream = fs.create(filePath); + outputStream.write(nonZeroByteArray(FILE_SIZE)); + outputStream.close(); + long uploadDurationMs = new Date().getTime() - start.getTime(); + + // The exact number of requests/responses that happen to create a file + // can vary - at the time of writing this code it takes 10 + // requests/responses for the 1000 byte file (33 for 100 MB), + // plus the initial container-check request but that + // can very easily change in the future. Just assert that we do roughly + // more than 2 but less than 15. + logOpResponseCount("Creating a 1K file", base); + base = assertWebResponsesInRange(base, 2, 15); + getBandwidthGaugeUpdater().triggerUpdate(true); + long bytesWritten = AzureMetricsTestUtil.getCurrentBytesWritten(getInstrumentation()); + assertTrue("The bytes written in the last second " + bytesWritten + + " is pretty far from the expected range of around " + FILE_SIZE + + " bytes plus a little overhead.", + bytesWritten > (FILE_SIZE / 2) && bytesWritten < (FILE_SIZE * 2)); + long totalBytesWritten = AzureMetricsTestUtil.getCurrentTotalBytesWritten(getInstrumentation()); + assertTrue("The total bytes written " + totalBytesWritten + + " is pretty far from the expected range of around " + FILE_SIZE + + " bytes plus a little overhead.", + totalBytesWritten >= FILE_SIZE && totalBytesWritten < (FILE_SIZE * 2)); + long uploadRate = AzureMetricsTestUtil.getLongGaugeValue(getInstrumentation(), WASB_UPLOAD_RATE); + System.out.println("Upload rate: " + uploadRate + " bytes/second."); + long expectedRate = (FILE_SIZE * 1000L) / uploadDurationMs; + assertTrue("The upload rate " + uploadRate + + " is below the expected range of around " + expectedRate + + " bytes/second that the unit test observed. This should never be" + + " the case since the test underestimates the rate by looking at " + + " end-to-end time instead of just block upload time.", + uploadRate >= expectedRate); + long uploadLatency = AzureMetricsTestUtil.getLongGaugeValue(getInstrumentation(), + WASB_UPLOAD_LATENCY); + System.out.println("Upload latency: " + uploadLatency); + long expectedLatency = uploadDurationMs; // We're uploading less than a block. + assertTrue("The upload latency " + uploadLatency + + " should be greater than zero now that I've just uploaded a file.", + uploadLatency > 0); + assertTrue("The upload latency " + uploadLatency + + " is more than the expected range of around " + expectedLatency + + " milliseconds that the unit test observed. This should never be" + + " the case since the test overestimates the latency by looking at " + + " end-to-end time instead of just block upload time.", + uploadLatency <= expectedLatency); + + // Read the file + start = new Date(); + InputStream inputStream = fs.open(filePath); + int count = 0; + while (inputStream.read() >= 0) { + count++; + } + inputStream.close(); + long downloadDurationMs = new Date().getTime() - start.getTime(); + assertEquals(FILE_SIZE, count); + + // Again, exact number varies. At the time of writing this code + // it takes 4 request/responses, so just assert a rough range between + // 1 and 10. + logOpResponseCount("Reading a 1K file", base); + base = assertWebResponsesInRange(base, 1, 10); + getBandwidthGaugeUpdater().triggerUpdate(false); + long totalBytesRead = AzureMetricsTestUtil.getCurrentTotalBytesRead(getInstrumentation()); + assertEquals(FILE_SIZE, totalBytesRead); + long bytesRead = AzureMetricsTestUtil.getCurrentBytesRead(getInstrumentation()); + assertTrue("The bytes read in the last second " + bytesRead + + " is pretty far from the expected range of around " + FILE_SIZE + + " bytes plus a little overhead.", + bytesRead > (FILE_SIZE / 2) && bytesRead < (FILE_SIZE * 2)); + long downloadRate = AzureMetricsTestUtil.getLongGaugeValue(getInstrumentation(), WASB_DOWNLOAD_RATE); + System.out.println("Download rate: " + downloadRate + " bytes/second."); + expectedRate = (FILE_SIZE * 1000L) / downloadDurationMs; + assertTrue("The download rate " + downloadRate + + " is below the expected range of around " + expectedRate + + " bytes/second that the unit test observed. This should never be" + + " the case since the test underestimates the rate by looking at " + + " end-to-end time instead of just block download time.", + downloadRate >= expectedRate); + long downloadLatency = AzureMetricsTestUtil.getLongGaugeValue(getInstrumentation(), + WASB_DOWNLOAD_LATENCY); + System.out.println("Download latency: " + downloadLatency); + expectedLatency = downloadDurationMs; // We're downloading less than a block. + assertTrue("The download latency " + downloadLatency + + " should be greater than zero now that I've just downloaded a file.", + downloadLatency > 0); + assertTrue("The download latency " + downloadLatency + + " is more than the expected range of around " + expectedLatency + + " milliseconds that the unit test observed. This should never be" + + " the case since the test overestimates the latency by looking at " + + " end-to-end time instead of just block download time.", + downloadLatency <= expectedLatency); + + assertNoErrors(); + } + + @Test + public void testMetricsOnBigFileCreateRead() throws Exception { + long base = getBaseWebResponses(); + + assertEquals(0, AzureMetricsTestUtil.getCurrentBytesWritten(getInstrumentation())); + + Path filePath = new Path("/metricsTest_webResponses"); + final int FILE_SIZE = 100 * 1024 * 1024; + + // Suppress auto-update of bandwidth metrics so we get + // to update them exactly when we want to. + getBandwidthGaugeUpdater().suppressAutoUpdate(); + + // Create a file + OutputStream outputStream = fs.create(filePath); + outputStream.write(new byte[FILE_SIZE]); + outputStream.close(); + + // The exact number of requests/responses that happen to create a file + // can vary - at the time of writing this code it takes 34 + // requests/responses for the 100 MB file, + // plus the initial container check request, but that + // can very easily change in the future. Just assert that we do roughly + // more than 20 but less than 50. + logOpResponseCount("Creating a 100 MB file", base); + base = assertWebResponsesInRange(base, 20, 50); + getBandwidthGaugeUpdater().triggerUpdate(true); + long totalBytesWritten = AzureMetricsTestUtil.getCurrentTotalBytesWritten(getInstrumentation()); + assertTrue("The total bytes written " + totalBytesWritten + + " is pretty far from the expected range of around " + FILE_SIZE + + " bytes plus a little overhead.", + totalBytesWritten >= FILE_SIZE && totalBytesWritten < (FILE_SIZE * 2)); + long uploadRate = AzureMetricsTestUtil.getLongGaugeValue(getInstrumentation(), WASB_UPLOAD_RATE); + System.out.println("Upload rate: " + uploadRate + " bytes/second."); + long uploadLatency = AzureMetricsTestUtil.getLongGaugeValue(getInstrumentation(), + WASB_UPLOAD_LATENCY); + System.out.println("Upload latency: " + uploadLatency); + assertTrue("The upload latency " + uploadLatency + + " should be greater than zero now that I've just uploaded a file.", + uploadLatency > 0); + + // Read the file + InputStream inputStream = fs.open(filePath); + int count = 0; + while (inputStream.read() >= 0) { + count++; + } + inputStream.close(); + assertEquals(FILE_SIZE, count); + + // Again, exact number varies. At the time of writing this code + // it takes 27 request/responses, so just assert a rough range between + // 20 and 40. + logOpResponseCount("Reading a 100 MB file", base); + base = assertWebResponsesInRange(base, 20, 40); + getBandwidthGaugeUpdater().triggerUpdate(false); + long totalBytesRead = AzureMetricsTestUtil.getCurrentTotalBytesRead(getInstrumentation()); + assertEquals(FILE_SIZE, totalBytesRead); + long downloadRate = AzureMetricsTestUtil.getLongGaugeValue(getInstrumentation(), WASB_DOWNLOAD_RATE); + System.out.println("Download rate: " + downloadRate + " bytes/second."); + long downloadLatency = AzureMetricsTestUtil.getLongGaugeValue(getInstrumentation(), + WASB_DOWNLOAD_LATENCY); + System.out.println("Download latency: " + downloadLatency); + assertTrue("The download latency " + downloadLatency + + " should be greater than zero now that I've just downloaded a file.", + downloadLatency > 0); + } + + @Test + public void testMetricsOnFileRename() throws Exception { + long base = getBaseWebResponses(); + + Path originalPath = new Path("/metricsTest_RenameStart"); + Path destinationPath = new Path("/metricsTest_RenameFinal"); + + // Create an empty file + assertEquals(0, AzureMetricsTestUtil.getLongCounterValue(getInstrumentation(), WASB_FILES_CREATED)); + assertTrue(fs.createNewFile(originalPath)); + logOpResponseCount("Creating an empty file", base); + base = assertWebResponsesInRange(base, 2, 20); + assertEquals(1, AzureMetricsTestUtil.getLongCounterValue(getInstrumentation(), WASB_FILES_CREATED)); + + // Rename the file + assertTrue(fs.rename(originalPath, destinationPath)); + // Varies: at the time of writing this code it takes 7 requests/responses. + logOpResponseCount("Renaming a file", base); + base = assertWebResponsesInRange(base, 2, 15); + + assertNoErrors(); + } + + @Test + public void testMetricsOnFileExistsDelete() throws Exception { + long base = getBaseWebResponses(); + + Path filePath = new Path("/metricsTest_delete"); + + // Check existence + assertFalse(fs.exists(filePath)); + // At the time of writing this code it takes 2 requests/responses to + // check existence, which seems excessive, plus initial request for + // container check. + logOpResponseCount("Checking file existence for non-existent file", base); + base = assertWebResponsesInRange(base, 1, 3); + + // Create an empty file + assertTrue(fs.createNewFile(filePath)); + base = getCurrentWebResponses(); + + // Check existence again + assertTrue(fs.exists(filePath)); + logOpResponseCount("Checking file existence for existent file", base); + base = assertWebResponsesInRange(base, 1, 2); + + // Delete the file + assertEquals(0, AzureMetricsTestUtil.getLongCounterValue(getInstrumentation(), WASB_FILES_DELETED)); + assertTrue(fs.delete(filePath, false)); + // At the time of writing this code it takes 4 requests/responses to + // delete, which seems excessive. Check for range 1-4 for now. + logOpResponseCount("Deleting a file", base); + base = assertWebResponsesInRange(base, 1, 4); + assertEquals(1, AzureMetricsTestUtil.getLongCounterValue(getInstrumentation(), WASB_FILES_DELETED)); + + assertNoErrors(); + } + + @Test + public void testMetricsOnDirRename() throws Exception { + long base = getBaseWebResponses(); + + Path originalDirName = new Path("/metricsTestDirectory_RenameStart"); + Path innerFileName = new Path(originalDirName, "innerFile"); + Path destDirName = new Path("/metricsTestDirectory_RenameFinal"); + + // Create an empty directory + assertTrue(fs.mkdirs(originalDirName)); + base = getCurrentWebResponses(); + + // Create an inner file + assertTrue(fs.createNewFile(innerFileName)); + base = getCurrentWebResponses(); + + // Rename the directory + assertTrue(fs.rename(originalDirName, destDirName)); + // At the time of writing this code it takes 11 requests/responses + // to rename the directory with one file. Check for range 1-20 for now. + logOpResponseCount("Renaming a directory", base); + base = assertWebResponsesInRange(base, 1, 20); + + assertNoErrors(); + } + + @Test + public void testClientErrorMetrics() throws Exception { + String directoryName = "metricsTestDirectory_ClientError"; + Path directoryPath = new Path("/" + directoryName); + assertTrue(fs.mkdirs(directoryPath)); + String leaseID = testAccount.acquireShortLease(directoryName); + try { + try { + fs.delete(directoryPath, true); + assertTrue("Should've thrown.", false); + } catch (AzureException ex) { + assertTrue("Unexpected exception: " + ex, + ex.getMessage().contains("lease")); + } + assertEquals(1, AzureMetricsTestUtil.getLongCounterValue(getInstrumentation(), WASB_CLIENT_ERRORS)); + assertEquals(0, AzureMetricsTestUtil.getLongCounterValue(getInstrumentation(), WASB_SERVER_ERRORS)); + } finally { + testAccount.releaseLease(leaseID, directoryName); + } + } + + private void logOpResponseCount(String opName, long base) { + System.out.println(opName + " took " + (getCurrentWebResponses() - base) + + " web responses to complete."); + } + + /** + * Gets (and asserts) the value of the wasb_web_responses counter just + * after the creation of the file system object. + */ + private long getBaseWebResponses() { + // The number of requests should start at 0 + return assertWebResponsesEquals(0, 0); + } + + /** + * Gets the current value of the wasb_web_responses counter. + */ + private long getCurrentWebResponses() { + return AzureMetricsTestUtil.getCurrentWebResponses(getInstrumentation()); + } + + /** + * Checks that the wasb_web_responses counter is at the given value. + * @param base The base value (before the operation of interest). + * @param expected The expected value for the operation of interest. + * @return The new base value now. + */ + private long assertWebResponsesEquals(long base, long expected) { + assertCounter(WASB_WEB_RESPONSES, base + expected, getMyMetrics()); + return base + expected; + } + + private void assertNoErrors() { + assertEquals(0, AzureMetricsTestUtil.getLongCounterValue(getInstrumentation(), WASB_CLIENT_ERRORS)); + assertEquals(0, AzureMetricsTestUtil.getLongCounterValue(getInstrumentation(), WASB_SERVER_ERRORS)); + } + + /** + * Checks that the wasb_web_responses counter is in the given range. + * @param base The base value (before the operation of interest). + * @param inclusiveLowerLimit The lower limit for what it should increase by. + * @param inclusiveUpperLimit The upper limit for what it should increase by. + * @return The new base value now. + */ + private long assertWebResponsesInRange(long base, + long inclusiveLowerLimit, + long inclusiveUpperLimit) { + long currentResponses = getCurrentWebResponses(); + long justOperation = currentResponses - base; + assertTrue(String.format( + "Web responses expected in range [%d, %d], but was %d.", + inclusiveLowerLimit, inclusiveUpperLimit, justOperation), + justOperation >= inclusiveLowerLimit && + justOperation <= inclusiveUpperLimit); + return currentResponses; + } + + /** + * Gets the metrics for the file system object. + * @return The metrics record. + */ + private MetricsRecordBuilder getMyMetrics() { + return getMetrics(getInstrumentation()); + } + + private AzureFileSystemInstrumentation getInstrumentation() { + return ((NativeAzureFileSystem)fs).getInstrumentation(); + } + + /** + * A matcher class for asserting that we got a tag with a given + * value. + */ + private static class TagMatcher extends TagExistsMatcher { + private final String tagValue; + + public TagMatcher(String tagName, String tagValue) { + super(tagName); + this.tagValue = tagValue; + } + + @Override + public boolean matches(MetricsTag toMatch) { + return toMatch.value().equals(tagValue); + } + + @Override + public void describeTo(Description desc) { + super.describeTo(desc); + desc.appendText(" with value " + tagValue); + } + } + + /** + * A matcher class for asserting that we got a tag with any value. + */ + private static class TagExistsMatcher extends BaseMatcher { + private final String tagName; + + public TagExistsMatcher(String tagName) { + this.tagName = tagName; + } + + @Override + public boolean matches(Object toMatch) { + MetricsTag asTag = (MetricsTag)toMatch; + return asTag.name().equals(tagName) && matches(asTag); + } + + protected boolean matches(MetricsTag toMatch) { + return true; + } + + @Override + public void describeTo(Description desc) { + desc.appendText("Has tag " + tagName); + } + } + +} diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestBandwidthGaugeUpdater.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestBandwidthGaugeUpdater.java new file mode 100644 index 00000000000..ef3442227f3 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestBandwidthGaugeUpdater.java @@ -0,0 +1,125 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure.metrics; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeNotNull; + +import java.util.Date; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount; +import org.junit.Assume; +import org.junit.Test; + +public class TestBandwidthGaugeUpdater { + @Test + public void testSingleThreaded() throws Exception { + AzureFileSystemInstrumentation instrumentation = + new AzureFileSystemInstrumentation(new Configuration()); + BandwidthGaugeUpdater updater = + new BandwidthGaugeUpdater(instrumentation, 1000, true); + updater.triggerUpdate(true); + assertEquals(0, AzureMetricsTestUtil.getCurrentBytesWritten(instrumentation)); + updater.blockUploaded(new Date(), new Date(), 150); + updater.triggerUpdate(true); + assertEquals(150, AzureMetricsTestUtil.getCurrentBytesWritten(instrumentation)); + updater.blockUploaded(new Date(new Date().getTime() - 10000), + new Date(), 200); + updater.triggerUpdate(true); + long currentBytes = AzureMetricsTestUtil.getCurrentBytesWritten(instrumentation); + assertTrue( + "We expect around (200/10 = 20) bytes written as the gauge value." + + "Got " + currentBytes, + currentBytes > 18 && currentBytes < 22); + updater.close(); + } + + @Test + public void testMultiThreaded() throws Exception { + final AzureFileSystemInstrumentation instrumentation = + new AzureFileSystemInstrumentation(new Configuration()); + final BandwidthGaugeUpdater updater = + new BandwidthGaugeUpdater(instrumentation, 1000, true); + Thread[] threads = new Thread[10]; + for (int i = 0; i < threads.length; i++) { + threads[i] = new Thread(new Runnable() { + @Override + public void run() { + updater.blockDownloaded(new Date(), new Date(), 10); + updater.blockDownloaded(new Date(0), new Date(0), 10); + } + }); + } + for (Thread t : threads) { + t.start(); + } + for (Thread t : threads) { + t.join(); + } + updater.triggerUpdate(false); + assertEquals(10 * threads.length, AzureMetricsTestUtil.getCurrentBytesRead(instrumentation)); + updater.close(); + } + + @Test + public void testFinalizerThreadShutdown() throws Exception { + + // force cleanup of any existing wasb filesystems + System.gc(); + System.runFinalization(); + + int nUpdaterThreadsStart = getWasbThreadCount(); + assertTrue("Existing WASB threads have not been cleared", nUpdaterThreadsStart == 0); + + final int nFilesystemsToSpawn = 10; + AzureBlobStorageTestAccount testAccount = null; + + for(int i = 0; i < nFilesystemsToSpawn; i++){ + testAccount = AzureBlobStorageTestAccount.createMock(); + testAccount.getFileSystem(); + } + + int nUpdaterThreadsAfterSpawn = getWasbThreadCount(); + Assume.assumeTrue("Background threads should have spawned.", nUpdaterThreadsAfterSpawn == 10); + + testAccount = null; //clear the last reachable reference + + // force cleanup + System.gc(); + System.runFinalization(); + + int nUpdaterThreadsAfterCleanup = getWasbThreadCount(); + assertTrue("Finalizers should have reduced the thread count. ", nUpdaterThreadsAfterCleanup == 0 ); + } + + private int getWasbThreadCount() { + int c = 0; + Map stacksStart = Thread.getAllStackTraces(); + for (Thread t : stacksStart.keySet()){ + if(t.getName().equals(BandwidthGaugeUpdater.THREAD_NAME)) + { + c++; + } + } + return c; + } +} diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestNativeAzureFileSystemMetricsSystem.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestNativeAzureFileSystemMetricsSystem.java new file mode 100644 index 00000000000..f44613d91d4 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestNativeAzureFileSystemMetricsSystem.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure.metrics; + +import static org.junit.Assert.*; + +import org.apache.hadoop.fs.*; +import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount; +import org.apache.hadoop.fs.azure.NativeAzureFileSystem; +import org.junit.*; + +/** + * Tests that the WASB-specific metrics system is working correctly. + */ +public class TestNativeAzureFileSystemMetricsSystem { + private static final String WASB_FILES_CREATED = "wasb_files_created"; + + private static int getFilesCreated(AzureBlobStorageTestAccount testAccount) { + return testAccount.getLatestMetricValue(WASB_FILES_CREATED, 0).intValue(); + } + + /** + * Tests that when we have multiple file systems created/destroyed + * metrics from each are published correctly. + * @throws Exception + */ + @Test + public void testMetricsAcrossFileSystems() + throws Exception { + AzureBlobStorageTestAccount a1, a2, a3; + + a1 = AzureBlobStorageTestAccount.createMock(); + assertEquals(0, getFilesCreated(a1)); + a2 = AzureBlobStorageTestAccount.createMock(); + assertEquals(0, getFilesCreated(a2)); + a1.getFileSystem().create(new Path("/foo")).close(); + a1.getFileSystem().create(new Path("/bar")).close(); + a2.getFileSystem().create(new Path("/baz")).close(); + assertEquals(0, getFilesCreated(a1)); + assertEquals(0, getFilesCreated(a2)); + a1.closeFileSystem(); // Causes the file system to close, which publishes metrics + a2.closeFileSystem(); + + assertEquals(2, getFilesCreated(a1)); + assertEquals(1, getFilesCreated(a2)); + a3 = AzureBlobStorageTestAccount.createMock(); + assertEquals(0, getFilesCreated(a3)); + a3.closeFileSystem(); + assertEquals(0, getFilesCreated(a3)); + } + + + @Test + public void testMetricsSourceNames() { + String name1 = NativeAzureFileSystem.newMetricsSourceName(); + String name2 = NativeAzureFileSystem.newMetricsSourceName(); + assertTrue(name1.startsWith("AzureFileSystemMetrics")); + assertTrue(name2.startsWith("AzureFileSystemMetrics")); + assertTrue(!name1.equals(name2)); + } +} diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestRollingWindowAverage.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestRollingWindowAverage.java new file mode 100644 index 00000000000..9b1fb8e60e1 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestRollingWindowAverage.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure.metrics; + +import static org.junit.Assert.*; +import org.junit.*; + +public class TestRollingWindowAverage { + /** + * Tests the basic functionality of the class. + */ + @Test + public void testBasicFunctionality() throws Exception { + RollingWindowAverage average = new RollingWindowAverage(100); + assertEquals(0, average.getCurrentAverage()); // Nothing there yet. + average.addPoint(5); + assertEquals(5, average.getCurrentAverage()); // One point in there. + Thread.sleep(50); + average.addPoint(15); + assertEquals(10, average.getCurrentAverage()); // Two points in there. + Thread.sleep(60); + assertEquals(15, average.getCurrentAverage()); // One point retired. + Thread.sleep(50); + assertEquals(0, average.getCurrentAverage()); // Both points retired. + } +} diff --git a/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml b/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml index fb2aa20e4ab..7eeff92f33e 100644 --- a/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml +++ b/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml @@ -26,6 +26,7 @@ org.apache.hadoop.fs.azure.NativeAzureFileSystem + + hadoop.security.groups.cache.secs 300 diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCompositeGroupMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCompositeGroupMapping.java new file mode 100644 index 00000000000..79f56e065a9 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCompositeGroupMapping.java @@ -0,0 +1,185 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.security; + +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.junit.Test; + + +public class TestCompositeGroupMapping { + public static final Log LOG = LogFactory.getLog(TestCompositeGroupMapping.class); + private static Configuration conf = new Configuration(); + + private static class TestUser { + String name; + String group; + String group2; + + public TestUser(String name, String group) { + this.name = name; + this.group = group; + } + + public TestUser(String name, String group, String group2) { + this(name, group); + this.group2 = group2; + } + }; + + private static TestUser john = new TestUser("John", "user-group"); + private static TestUser hdfs = new TestUser("hdfs", "supergroup"); + private static TestUser jack = new TestUser("Jack", "user-group", "dev-group-1"); + + private static final String PROVIDER_SPECIFIC_CONF = ".test.prop"; + private static final String PROVIDER_SPECIFIC_CONF_KEY = + GroupMappingServiceProvider.GROUP_MAPPING_CONFIG_PREFIX + PROVIDER_SPECIFIC_CONF; + private static final String PROVIDER_SPECIFIC_CONF_VALUE_FOR_USER = "value-for-user"; + private static final String PROVIDER_SPECIFIC_CONF_VALUE_FOR_CLUSTER = "value-for-cluster"; + + private static abstract class GroupMappingProviderBase + implements GroupMappingServiceProvider, Configurable { + + private Configuration conf; + + @Override + public void setConf(Configuration conf) { + this.conf = conf; + } + + @Override + public Configuration getConf() { + return this.conf; + } + + @Override + public void cacheGroupsRefresh() throws IOException { + + } + + @Override + public void cacheGroupsAdd(List groups) throws IOException { + + } + + protected List toList(String group) { + if (group != null) { + return Arrays.asList(new String[] {group}); + } + return new ArrayList(); + } + + protected void checkTestConf(String expectedValue) { + String configValue = getConf().get(PROVIDER_SPECIFIC_CONF_KEY); + if (configValue == null || !configValue.equals(expectedValue)) { + throw new RuntimeException("Failed to find mandatory configuration of " + PROVIDER_SPECIFIC_CONF_KEY); + } + } + }; + + private static class UserProvider extends GroupMappingProviderBase { + @Override + public List getGroups(String user) throws IOException { + checkTestConf(PROVIDER_SPECIFIC_CONF_VALUE_FOR_USER); + + String group = null; + if (user.equals(john.name)) { + group = john.group; + } else if (user.equals(jack.name)) { + group = jack.group; + } + + return toList(group); + } + } + + private static class ClusterProvider extends GroupMappingProviderBase { + @Override + public List getGroups(String user) throws IOException { + checkTestConf(PROVIDER_SPECIFIC_CONF_VALUE_FOR_CLUSTER); + + String group = null; + if (user.equals(hdfs.name)) { + group = hdfs.group; + } else if (user.equals(jack.name)) { // jack has another group from clusterProvider + group = jack.group2; + } + + return toList(group); + } + } + + static { + conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, + CompositeGroupsMapping.class, GroupMappingServiceProvider.class); + conf.set(CompositeGroupsMapping.MAPPING_PROVIDERS_CONFIG_KEY, "userProvider,clusterProvider"); + + conf.setClass(CompositeGroupsMapping.MAPPING_PROVIDER_CONFIG_PREFIX + ".userProvider", + UserProvider.class, GroupMappingServiceProvider.class); + + conf.setClass(CompositeGroupsMapping.MAPPING_PROVIDER_CONFIG_PREFIX + ".clusterProvider", + ClusterProvider.class, GroupMappingServiceProvider.class); + + conf.set(CompositeGroupsMapping.MAPPING_PROVIDER_CONFIG_PREFIX + + ".clusterProvider" + PROVIDER_SPECIFIC_CONF, PROVIDER_SPECIFIC_CONF_VALUE_FOR_CLUSTER); + + conf.set(CompositeGroupsMapping.MAPPING_PROVIDER_CONFIG_PREFIX + + ".userProvider" + PROVIDER_SPECIFIC_CONF, PROVIDER_SPECIFIC_CONF_VALUE_FOR_USER); + } + + @Test + public void TestMultipleGroupsMapping() throws Exception { + Groups groups = new Groups(conf); + + assertTrue(groups.getGroups(john.name).get(0).equals(john.group)); + assertTrue(groups.getGroups(hdfs.name).get(0).equals(hdfs.group)); + } + + @Test + public void TestMultipleGroupsMappingWithCombined() throws Exception { + conf.set(CompositeGroupsMapping.MAPPING_PROVIDERS_COMBINED_CONFIG_KEY, "true"); + Groups groups = new Groups(conf); + + assertTrue(groups.getGroups(jack.name).size() == 2); + // the configured providers list in order is "userProvider,clusterProvider" + // group -> userProvider, group2 -> clusterProvider + assertTrue(groups.getGroups(jack.name).contains(jack.group)); + assertTrue(groups.getGroups(jack.name).contains(jack.group2)); + } + + @Test + public void TestMultipleGroupsMappingWithoutCombined() throws Exception { + conf.set(CompositeGroupsMapping.MAPPING_PROVIDERS_COMBINED_CONFIG_KEY, "false"); + Groups groups = new Groups(conf); + + // the configured providers list in order is "userProvider,clusterProvider" + // group -> userProvider, group2 -> clusterProvider + assertTrue(groups.getGroups(jack.name).size() == 1); + assertTrue(groups.getGroups(jack.name).get(0).equals(jack.group)); + } +} From cfd9344540d4d0f44fd5b4c450ce3706e60045d4 Mon Sep 17 00:00:00 2001 From: Jason Darrell Lowe Date: Thu, 26 Jun 2014 19:56:09 +0000 Subject: [PATCH 053/112] MAPREDUCE-5939. StartTime showing up as the epoch time in JHS UI after upgrade. Contributed by Chen He git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1605892 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../v2/jobhistory/FileNameIndexUtils.java | 10 ++++--- .../v2/jobhistory/TestFileNameIndexUtils.java | 27 +++++++++++++++++++ 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 1220895908b..e3a72b8046f 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -273,6 +273,9 @@ Release 2.5.0 - UNRELEASED MAPREDUCE-5924. Changed TaskAttemptImpl to ignore TA_COMMIT_PENDING event at COMMIT_PENDING state. (Zhijie Shen via jianhe) + MAPREDUCE-5939. StartTime showing up as the epoch time in JHS UI after + upgrade (Chen He via jlowe) + Release 2.4.1 - 2014-06-23 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java index 54bbcd93c52..741da11b4ae 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java @@ -168,10 +168,14 @@ public static JobIndexInfo getIndexInfo(String jhFileName) throws IOException { decodeJobHistoryFileName(jobDetails[QUEUE_NAME_INDEX])); try{ - indexInfo.setJobStartTime( - Long.parseLong(decodeJobHistoryFileName(jobDetails[JOB_START_TIME_INDEX]))); + if (jobDetails.length <= JOB_START_TIME_INDEX) { + indexInfo.setJobStartTime(indexInfo.getSubmitTime()); + } else { + indexInfo.setJobStartTime( + Long.parseLong(decodeJobHistoryFileName(jobDetails[JOB_START_TIME_INDEX]))); + } } catch (NumberFormatException e){ - LOG.warn("Unable to parse launch time from job history file " + LOG.warn("Unable to parse start time from job history file " + jhFileName + " : " + e); } } catch (IndexOutOfBoundsException e) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java index 015c87ad7f5..a0d5fceb35e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java @@ -39,6 +39,17 @@ public class TestFileNameIndexUtils { + FileNameIndexUtils.DELIMITER + "%s" + JobHistoryUtils.JOB_HISTORY_FILE_EXTENSION; + private static final String OLD_FORMAT_BEFORE_ADD_START_TIME = "%s" + + FileNameIndexUtils.DELIMITER + "%s" + + FileNameIndexUtils.DELIMITER + "%s" + + FileNameIndexUtils.DELIMITER + "%s" + + FileNameIndexUtils.DELIMITER + "%s" + + FileNameIndexUtils.DELIMITER + "%s" + + FileNameIndexUtils.DELIMITER + "%s" + + FileNameIndexUtils.DELIMITER + "%s" + + FileNameIndexUtils.DELIMITER + "%s" + + JobHistoryUtils.JOB_HISTORY_FILE_EXTENSION; + private static final String JOB_HISTORY_FILE_FORMATTER = "%s" + FileNameIndexUtils.DELIMITER + "%s" + FileNameIndexUtils.DELIMITER + "%s" @@ -235,6 +246,22 @@ public void testQueueNamePercentDecoding() throws IOException { QUEUE_NAME_WITH_DELIMITER, info.getQueueName()); } + @Test + public void testJobStartTimeBackwardsCompatible() throws IOException{ + String jobHistoryFile = String.format(OLD_FORMAT_BEFORE_ADD_START_TIME, + JOB_ID, + SUBMIT_TIME, + USER_NAME, + JOB_NAME_WITH_DELIMITER_ESCAPE, + FINISH_TIME, + NUM_MAPS, + NUM_REDUCES, + JOB_STATUS, + QUEUE_NAME ); + JobIndexInfo info = FileNameIndexUtils.getIndexInfo(jobHistoryFile); + Assert.assertEquals(info.getJobStartTime(), info.getSubmitTime()); + } + @Test public void testJobHistoryFileNameBackwardsCompatible() throws IOException { JobID oldJobId = JobID.forName(JOB_ID); From d43112471b2b8abe9a7abbd452e1144ee63a40ef Mon Sep 17 00:00:00 2001 From: Chris Nauroth Date: Thu, 26 Jun 2014 22:58:21 +0000 Subject: [PATCH 054/112] HADOOP-10754. Reenable several HA ZooKeeper-related tests on Windows. Contributed by Chris Nauroth. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1605924 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../org/apache/hadoop/ha/TestActiveStandbyElectorRealZK.java | 4 ---- .../org/apache/hadoop/ha/TestZKFailoverControllerStress.java | 5 ----- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 52cf7413a72..a403f8c607d 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -477,6 +477,9 @@ Release 2.5.0 - UNRELEASED HADOOP-10674. Improve PureJavaCrc32 performance and use java.util.zip.CRC32 for Java 7 and above. (szetszwo) + HADOOP-10754. Reenable several HA ZooKeeper-related tests on Windows. + (cnauroth) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestActiveStandbyElectorRealZK.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestActiveStandbyElectorRealZK.java index 22d0e20de82..fb8c236c8fd 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestActiveStandbyElectorRealZK.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestActiveStandbyElectorRealZK.java @@ -20,7 +20,6 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import static org.junit.Assume.assumeTrue; import java.util.Collections; import java.util.UUID; @@ -30,7 +29,6 @@ import org.apache.hadoop.ha.ActiveStandbyElector.ActiveStandbyElectorCallback; import org.apache.hadoop.ha.ActiveStandbyElector.State; import org.apache.hadoop.util.ZKUtil.ZKAuthInfo; -import org.apache.hadoop.util.Shell; import org.apache.log4j.Level; import org.apache.zookeeper.ZooDefs.Ids; import org.apache.zookeeper.server.ZooKeeperServer; @@ -62,8 +60,6 @@ public class TestActiveStandbyElectorRealZK extends ClientBaseWithFixes { @Override public void setUp() throws Exception { - // skip tests on Windows until after resolution of ZooKeeper client bug - assumeTrue(!Shell.WINDOWS); super.setUp(); zkServer = getServer(serverFactory); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestZKFailoverControllerStress.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestZKFailoverControllerStress.java index a3b8ec6ab37..bdbf1d9c2c2 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestZKFailoverControllerStress.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestZKFailoverControllerStress.java @@ -17,14 +17,11 @@ */ package org.apache.hadoop.ha; -import static org.junit.Assume.assumeTrue; - import java.util.Random; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; -import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.Time; import org.junit.After; import org.junit.Before; @@ -49,8 +46,6 @@ public class TestZKFailoverControllerStress extends ClientBaseWithFixes { @Before public void setupConfAndServices() throws Exception { - // skip tests on Windows until after resolution of ZooKeeper client bug - assumeTrue(!Shell.WINDOWS); conf = new Configuration(); conf.set(ZKFailoverController.ZK_QUORUM_KEY, hostPort); this.cluster = new MiniZKFCCluster(conf, getServer(serverFactory)); From 280bdb9a16a898118421aee16db11f52eed9bdae Mon Sep 17 00:00:00 2001 From: Chris Nauroth Date: Thu, 26 Jun 2014 23:12:45 +0000 Subject: [PATCH 055/112] HDFS-6572. Add an option to the NameNode that prints the software and on-disk image versions. Contributed by Charles Lamb. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1605928 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../server/common/HdfsServerConstants.java | 3 +- .../hadoop/hdfs/server/namenode/FSImage.java | 13 ++++ .../hadoop/hdfs/server/namenode/NameNode.java | 49 +++++++++--- .../namenode/TestMetadataVersionOutput.java | 77 +++++++++++++++++++ 5 files changed, 132 insertions(+), 13 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetadataVersionOutput.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index b3738dd2d61..e3ca235b335 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -479,6 +479,9 @@ Release 2.5.0 - UNRELEASED HDFS-6595. Allow the maximum threads for balancing on datanodes to be configurable. (Benoy Antony via szetszwo) + HDFS-6572. Add an option to the NameNode that prints the software and + on-disk image versions. (Charles Lamb via cnauroth) + OPTIMIZATIONS HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java index 7a83bbf21e9..486d426bab1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java @@ -92,7 +92,8 @@ static public enum StartupOption{ RECOVER ("-recover"), FORCE("-force"), NONINTERACTIVE("-nonInteractive"), - RENAMERESERVED("-renameReserved"); + RENAMERESERVED("-renameReserved"), + METADATAVERSION("-metadataVersion"); private static final Pattern ENUM_WITH_ROLLING_UPGRADE_OPTION = Pattern.compile( "(\\w+)\\((\\w+)\\)"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java index d99985e5dc9..09d81f66fd7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java @@ -214,6 +214,13 @@ boolean recoverTransitionRead(StartupOption startOpt, FSNamesystem target, int layoutVersion = storage.getLayoutVersion(); + if (startOpt == StartupOption.METADATAVERSION) { + System.out.println("HDFS Image Version: " + layoutVersion); + System.out.println("Software format version: " + + HdfsConstants.NAMENODE_LAYOUT_VERSION); + return false; + } + if (layoutVersion < Storage.LAST_PRE_UPGRADE_LAYOUT_VERSION) { NNStorage.checkVersionUpgradable(storage.getLayoutVersion()); } @@ -289,6 +296,12 @@ private boolean recoverStorageDirs(StartupOption startOpt, storage.dirIterator(); it.hasNext();) { StorageDirectory sd = it.next(); StorageState curState; + if (startOpt == StartupOption.METADATAVERSION) { + /* All we need is the layout version. */ + storage.readProperties(sd); + return true; + } + try { curState = sd.analyzeStorage(startOpt, storage); // sd is locked but not opened diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index 110e020d18f..645b5997925 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -201,25 +201,28 @@ public static enum OperationCategory { }; private static final String USAGE = "Usage: java NameNode [" - + StartupOption.BACKUP.getName() + "] | [" - + StartupOption.CHECKPOINT.getName() + "] | [" + + StartupOption.BACKUP.getName() + "] | \n\t[" + + StartupOption.CHECKPOINT.getName() + "] | \n\t[" + StartupOption.FORMAT.getName() + " [" + StartupOption.CLUSTERID.getName() + " cid ] [" + StartupOption.FORCE.getName() + "] [" - + StartupOption.NONINTERACTIVE.getName() + "] ] | [" + + StartupOption.NONINTERACTIVE.getName() + "] ] | \n\t[" + StartupOption.UPGRADE.getName() + " [" + StartupOption.CLUSTERID.getName() + " cid]" + - " [" + StartupOption.RENAMERESERVED.getName() + "] ] | [" - + StartupOption.ROLLBACK.getName() + "] | [" + " [" + StartupOption.RENAMERESERVED.getName() + "] ] | \n\t[" + + StartupOption.ROLLBACK.getName() + "] | \n\t[" + StartupOption.ROLLINGUPGRADE.getName() + " <" + RollingUpgradeStartupOption.DOWNGRADE.name().toLowerCase() + "|" - + RollingUpgradeStartupOption.ROLLBACK.name().toLowerCase() + "> ] | [" - + StartupOption.FINALIZE.getName() + "] | [" - + StartupOption.IMPORT.getName() + "] | [" - + StartupOption.INITIALIZESHAREDEDITS.getName() + "] | [" - + StartupOption.BOOTSTRAPSTANDBY.getName() + "] | [" - + StartupOption.RECOVER.getName() + " [ " + StartupOption.FORCE.getName() - + " ] ]"; + + RollingUpgradeStartupOption.ROLLBACK.name().toLowerCase() + "> ] | \n\t[" + + StartupOption.FINALIZE.getName() + "] | \n\t[" + + StartupOption.IMPORT.getName() + "] | \n\t[" + + StartupOption.INITIALIZESHAREDEDITS.getName() + "] | \n\t[" + + StartupOption.BOOTSTRAPSTANDBY.getName() + "] | \n\t[" + + StartupOption.RECOVER.getName() + " [ " + + StartupOption.FORCE.getName() + "] ] | \n\t[" + + StartupOption.METADATAVERSION.getName() + " ] " + + " ]"; + public long getProtocolVersion(String protocol, long clientVersion) throws IOException { @@ -1266,6 +1269,8 @@ static StartupOption parseArguments(String args[]) { "can't understand option \"" + args[i] + "\""); } } + } else if (StartupOption.METADATAVERSION.getName().equalsIgnoreCase(cmd)) { + startOpt = StartupOption.METADATAVERSION; } else { return null; } @@ -1318,6 +1323,21 @@ private static void doRecovery(StartupOption startOpt, Configuration conf) } } + /** + * Verify that configured directories exist, then print the metadata versions + * of the software and the image. + * + * @param conf configuration to use + * @throws IOException + */ + private static boolean printMetadataVersion(Configuration conf) + throws IOException { + final FSImage fsImage = new FSImage(conf); + final FSNamesystem fs = new FSNamesystem(conf, fsImage, false); + return fsImage.recoverTransitionRead( + StartupOption.METADATAVERSION, fs, null); + } + public static NameNode createNameNode(String argv[], Configuration conf) throws IOException { LOG.info("createNameNode " + Arrays.asList(argv)); @@ -1382,6 +1402,11 @@ public static NameNode createNameNode(String argv[], Configuration conf) NameNode.doRecovery(startOpt, conf); return null; } + case METADATAVERSION: { + printMetadataVersion(conf); + terminate(0); + return null; // avoid javac warning + } default: { DefaultMetricsSystem.initialize("NameNode"); return new NameNode(conf); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetadataVersionOutput.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetadataVersionOutput.java new file mode 100644 index 00000000000..0e809cf9c66 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetadataVersionOutput.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode; + +import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains; +import static org.junit.Assert.assertTrue; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; + +public class TestMetadataVersionOutput { + + private MiniDFSCluster dfsCluster = null; + private final Configuration conf = new Configuration(); + + @Before + public void setUp() throws Exception { + dfsCluster = new MiniDFSCluster.Builder(conf). + numDataNodes(1). + checkExitOnShutdown(false). + build(); + dfsCluster.waitClusterUp(); + } + + @After + public void tearDown() throws Exception { + if (dfsCluster != null) { + dfsCluster.shutdown(); + } + Thread.sleep(2000); + } + + @Test(timeout = 30000) + public void testMetadataVersionOutput() throws IOException { + + final PrintStream origOut = System.out; + final ByteArrayOutputStream baos = new ByteArrayOutputStream(); + final PrintStream stdOut = new PrintStream(baos); + System.setOut(stdOut); + try { + NameNode.createNameNode(new String[] { "-metadataVersion" }, conf); + } catch (Exception e) { + assertExceptionContains("ExitException", e); + } + /* Check if meta data version is printed correctly. */ + final String verNumStr = HdfsConstants.NAMENODE_LAYOUT_VERSION + ""; + assertTrue(baos.toString("UTF-8"). + contains("HDFS Image Version: " + verNumStr)); + assertTrue(baos.toString("UTF-8"). + contains("Software format version: " + verNumStr)); + System.setOut(origOut); + } +} From f084170a0a7fd6c304b8418e4efb764ed90f3096 Mon Sep 17 00:00:00 2001 From: Ravi Prakash Date: Fri, 27 Jun 2014 06:43:46 +0000 Subject: [PATCH 056/112] YARN 2163. WebUI: Order of AppId in apps table should be consistent with ApplicationId.compareTo() git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1605964 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../src/main/resources/webapps/static/yarn.dt.plugins.js | 8 +------- .../hadoop/yarn/server/resourcemanager/webapp/RmView.java | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 8cace35c206..6d17d6ae447 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -293,6 +293,9 @@ Release 2.5.0 - UNRELEASED YARN-2204. TestAMRestart#testAMRestartWithExistingContainers assumes CapacityScheduler. (Robert Kanter via kasha) + YARN-2163. WebUI: Order of AppId in apps table should be consistent with + ApplicationId.compareTo(). (Wangda Tan via raviprak) + Release 2.4.1 - 2014-06-23 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/yarn.dt.plugins.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/yarn.dt.plugins.js index 1179a7d195e..0c683e767de 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/yarn.dt.plugins.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/yarn.dt.plugins.js @@ -119,13 +119,7 @@ function parseHadoopID(data, type, full) { return data; } //Return the visible string rather than the entire HTML tag - if (type === 'filter') { - return data.split('>')[1].split('<')[0]; - } - //Parse the ID for 'sort', 'type' and undefined - //The number after the last '_' and before the end tag '<' - var splits = data.split('_'); - return splits[parseInt(splits.length-1)].split('<')[0]; + return data.split('>')[1].split('<')[0]; } //JSON array element is "20000 attempt_1360183373897_0001_m_000002_0" diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmView.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmView.java index 1f3de367e14..769c4da6921 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmView.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmView.java @@ -77,7 +77,7 @@ protected String getAppsTableColumnDefs() { StringBuilder sb = new StringBuilder(); return sb .append("[\n") - .append("{'sType':'numeric', 'aTargets': [0]") + .append("{'sType':'string', 'aTargets': [0]") .append(", 'mRender': parseHadoopID }") .append("\n, {'sType':'numeric', 'aTargets': [5, 6]") From fdf901451fbcb614826b2238cad13d00b49f2157 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Fri, 27 Jun 2014 08:32:51 +0000 Subject: [PATCH 057/112] HADOOP-10565. Support IP ranges (CIDR) in proxyuser.hosts. (Contributed by Benoy Antony) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1605987 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../DefaultImpersonationProvider.java | 49 +-- .../org/apache/hadoop/util/MachineList.java | 203 ++++++++++++ .../src/site/apt/SecureMode.apt.vm | 19 ++ .../security/authorize/TestProxyUsers.java | 92 ++++++ .../apache/hadoop/util/TestMachineList.java | 291 ++++++++++++++++++ 6 files changed, 621 insertions(+), 36 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestMachineList.java diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index a403f8c607d..3603d7ed6d2 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -480,6 +480,9 @@ Release 2.5.0 - UNRELEASED HADOOP-10754. Reenable several HA ZooKeeper-related tests on Windows. (cnauroth) + HADOOP-10565. Support IP ranges (CIDR) in proxyuser.hosts. (Benoy Antony + via Arpit Agarwal) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/DefaultImpersonationProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/DefaultImpersonationProvider.java index f6255733081..afa46582d32 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/DefaultImpersonationProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/DefaultImpersonationProvider.java @@ -18,8 +18,6 @@ package org.apache.hadoop.security.authorize; -import java.net.InetAddress; -import java.net.UnknownHostException; import java.util.Collection; import java.util.HashMap; import java.util.Map; @@ -28,7 +26,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.MachineList; import com.google.common.annotations.VisibleForTesting; @@ -46,8 +44,8 @@ public class DefaultImpersonationProvider implements ImpersonationProvider { // acl and list of hosts per proxyuser private Map proxyUserAcl = new HashMap(); - private Map> proxyHosts = - new HashMap>(); + private static Map proxyHosts = + new HashMap(); private Configuration conf; @Override @@ -70,7 +68,7 @@ public void setConf(Configuration conf) { allMatchKeys = conf.getValByRegex(CONF_HADOOP_PROXYUSER_RE_HOSTS); for(Entry entry : allMatchKeys.entrySet()) { proxyHosts.put(entry.getKey(), - StringUtils.getTrimmedStringCollection(entry.getValue())); + new MachineList(entry.getValue())); } } @@ -95,27 +93,10 @@ public void authorize(UserGroupInformation user, + " is not allowed to impersonate " + user.getUserName()); } - boolean ipAuthorized = false; - Collection ipList = proxyHosts.get( + MachineList MachineList = proxyHosts.get( getProxySuperuserIpConfKey(realUser.getShortUserName())); - if (isWildcardList(ipList)) { - ipAuthorized = true; - } else if (ipList != null && !ipList.isEmpty()) { - for (String allowedHost : ipList) { - InetAddress hostAddr; - try { - hostAddr = InetAddress.getByName(allowedHost); - } catch (UnknownHostException e) { - continue; - } - if (hostAddr.getHostAddress().equals(remoteAddress)) { - // Authorization is successful - ipAuthorized = true; - } - } - } - if(!ipAuthorized) { + if(!MachineList.includes(remoteAddress)) { throw new AuthorizationException("Unauthorized connection for super-user: " + realUser.getUserName() + " from IP " + remoteAddress); } @@ -128,16 +109,6 @@ private String getAclKey(String key) { } return key; } - - /** - * Return true if the configuration specifies the special configuration value - * "*", indicating that any group or host list is allowed to use this configuration. - */ - private boolean isWildcardList(Collection list) { - return (list != null) && - (list.size() == 1) && - (list.contains("*")); - } /** * Returns configuration key for effective usergroups allowed for a superuser @@ -180,6 +151,12 @@ public Map> getProxyGroups() { @VisibleForTesting public Map> getProxyHosts() { - return proxyHosts; + Map> tmpProxyHosts = + new HashMap>(); + for (Map.Entry proxyHostEntry :proxyHosts.entrySet()) { + tmpProxyHosts.put(proxyHostEntry.getKey(), + proxyHostEntry.getValue().getCollection()); + } + return tmpProxyHosts; } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java new file mode 100644 index 00000000000..7f2e2c8de7e --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/MachineList.java @@ -0,0 +1,203 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.util; + +import java.net.InetAddress; + +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.commons.net.util.SubnetUtils; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.net.InetAddresses; + +/** + * Container class which holds a list of ip/host addresses and + * answers membership queries. + * . + * Accepts list of ip addresses, ip addreses in CIDR format and/or + * host addresses. + */ + +public class MachineList { + + public static final Log LOG = LogFactory.getLog(MachineList.class); + + /** + * InetAddressFactory is used to obtain InetAddress from host. + * This class makes it easy to simulate host to ip mappings during testing. + * + */ + public static class InetAddressFactory { + + static final InetAddressFactory S_INSTANCE = new InetAddressFactory(); + + public InetAddress getByName (String host) throws UnknownHostException { + return InetAddress.getByName(host); + } + } + + private final boolean all; + private final Set ipAddresses; + private final List cidrAddresses; + private final Set hostNames; + private final InetAddressFactory addressFactory; + + /** + * + * @param hostEntries comma separated ip/cidr/host addresses + */ + public MachineList(String hostEntries) { + this(StringUtils.getTrimmedStringCollection(hostEntries), + InetAddressFactory.S_INSTANCE); + } + + /** + * Accepts a collection of ip/cidr/host addresses + * + * @param hostEntries + * @param addressFactory addressFactory to convert host to InetAddress + */ + public MachineList(Collection hostEntries, InetAddressFactory addressFactory) { + this.addressFactory = addressFactory; + if (hostEntries != null) { + if ((hostEntries.size() == 1) && (hostEntries.contains("*"))) { + all = true; + ipAddresses = null; + hostNames = null; + cidrAddresses = null; + } else { + all = false; + Set ips = new HashSet(); + List cidrs = new LinkedList(); + Set hosts = new HashSet(); + for (String hostEntry : hostEntries) { + //ip address range + if (hostEntry.indexOf("/") > -1) { + try { + SubnetUtils subnet = new SubnetUtils(hostEntry); + subnet.setInclusiveHostCount(true); + cidrs.add(subnet.getInfo()); + } catch (IllegalArgumentException e) { + LOG.warn("Invalid CIDR syntax : " + hostEntry); + throw e; + } + } else if (InetAddresses.isInetAddress(hostEntry)) { //ip address + ips.add(hostEntry); + } else { //hostname + hosts.add(hostEntry); + } + } + ipAddresses = (ips.size() > 0) ? ips : null; + cidrAddresses = (cidrs.size() > 0) ? cidrs : null; + hostNames = (hosts.size() > 0) ? hosts : null; + } + } else { + all = false; + ipAddresses = null; + hostNames = null; + cidrAddresses = null; + } + } + /** + * Accepts an ip address and return true if ipAddress is in the list + * @param ipAddress + * @return true if ipAddress is part of the list + */ + public boolean includes(String ipAddress) { + + if (all) { + return true; + } + + //check in the set of ipAddresses + if ((ipAddresses != null) && ipAddresses.contains(ipAddress)) { + return true; + } + + //iterate through the ip ranges for inclusion + if (cidrAddresses != null) { + for(SubnetUtils.SubnetInfo cidrAddress : cidrAddresses) { + if(cidrAddress.isInRange(ipAddress)) { + return true; + } + } + } + + //check if the ipAddress matches one of hostnames + if (hostNames != null) { + //convert given ipAddress to hostname and look for a match + InetAddress hostAddr; + try { + hostAddr = addressFactory.getByName(ipAddress); + if ((hostAddr != null) && hostNames.contains(hostAddr.getCanonicalHostName())) { + return true; + } + } catch (UnknownHostException e) { + //ignore the exception and proceed to resolve the list of hosts + } + + //loop through host addresses and convert them to ip and look for a match + for (String host : hostNames) { + try { + hostAddr = addressFactory.getByName(host); + } catch (UnknownHostException e) { + continue; + } + if (hostAddr.getHostAddress().equals(ipAddress)) { + return true; + } + } + } + return false; + } + + /** + * returns the contents of the MachineList as a Collection + * This can be used for testing + * @return contents of the MachineList + */ + @VisibleForTesting + public Collection getCollection() { + Collection list = new ArrayList(); + if (all) { + list.add("*"); + } else { + if (ipAddresses != null) { + list.addAll(ipAddresses); + } + if (hostNames != null) { + list.addAll(hostNames); + } + if (cidrAddresses != null) { + for(SubnetUtils.SubnetInfo cidrAddress : cidrAddresses) { + list.add(cidrAddress.getCidrSignature()); + } + } + } + return list; + } +} diff --git a/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm index 99755e5638d..c02ea39d718 100644 --- a/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm +++ b/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm @@ -236,6 +236,25 @@ KVNO Timestamp Principal ---- + The <<>> accepts list of ip addresses, + ip address ranges in CIDR format and/or host names. + + For example, by specifying as below in core-site.xml, + user named <<>> accessing from hosts in the range + 10.222.0.0-15 and 10.113.221.221 + can impersonate any user belonging to any group. + + ---- + + hadoop.proxyuser.oozie.hosts + 10.222.0.0/16,10.113.221.221 + + + hadoop.proxyuser.oozie.groups + * + +---- + ** Secure DataNode Because the data transfer protocol of DataNode diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestProxyUsers.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestProxyUsers.java index 670a512230d..3823d70391a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestProxyUsers.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestProxyUsers.java @@ -21,6 +21,7 @@ import static org.junit.Assert.fail; import java.io.IOException; +import java.security.SecureRandom; import java.util.Arrays; import java.util.Collection; @@ -50,6 +51,7 @@ public class TestProxyUsers { private static final String[] SUDO_GROUP_NAMES = new String[] { "sudo_proxied_user" }; private static final String PROXY_IP = "1.2.3.4"; + private static final String PROXY_IP_RANGE = "10.222.0.0/16,10.113.221.221"; /** * Test the netgroups (groups in ACL rules that start with @) @@ -294,6 +296,29 @@ public void testWildcardIP() { assertNotAuthorized(proxyUserUgi, "1.2.3.4"); assertNotAuthorized(proxyUserUgi, "1.2.3.5"); } + + @Test + public void testIPRange() { + Configuration conf = new Configuration(); + conf.set( + DefaultImpersonationProvider.getProxySuperuserGroupConfKey(REAL_USER_NAME), + "*"); + conf.set( + DefaultImpersonationProvider.getProxySuperuserIpConfKey(REAL_USER_NAME), + PROXY_IP_RANGE); + ProxyUsers.refreshSuperUserGroupsConfiguration(conf); + + // First try proxying a group that's allowed + UserGroupInformation realUserUgi = UserGroupInformation + .createRemoteUser(REAL_USER_NAME); + UserGroupInformation proxyUserUgi = UserGroupInformation.createProxyUserForTesting( + PROXY_USER_NAME, realUserUgi, GROUP_NAMES); + + // From good IP + assertAuthorized(proxyUserUgi, "10.222.0.0"); + // From bad IP + assertNotAuthorized(proxyUserUgi, "10.221.0.0"); + } @Test public void testWithDuplicateProxyGroups() throws Exception { @@ -431,4 +456,71 @@ public Configuration getConf() { return null; } } + + public static void loadTest(String ipString, int testRange) { + Configuration conf = new Configuration(); + conf.set( + DefaultImpersonationProvider.getProxySuperuserGroupConfKey(REAL_USER_NAME), + StringUtils.join(",", Arrays.asList(GROUP_NAMES))); + + conf.set( + DefaultImpersonationProvider.getProxySuperuserIpConfKey(REAL_USER_NAME), + ipString + ); + ProxyUsers.refreshSuperUserGroupsConfiguration(conf); + + + // First try proxying a group that's allowed + UserGroupInformation realUserUgi = UserGroupInformation + .createRemoteUser(REAL_USER_NAME); + UserGroupInformation proxyUserUgi = UserGroupInformation.createProxyUserForTesting( + PROXY_USER_NAME, realUserUgi, GROUP_NAMES); + + long startTime = System.nanoTime(); + SecureRandom sr = new SecureRandom(); + for (int i=1; i < 1000000; i++){ + try { + ProxyUsers.authorize(proxyUserUgi, "1.2.3."+ sr.nextInt(testRange)); + } catch (AuthorizationException e) { + } + } + long stopTime = System.nanoTime(); + long elapsedTime = stopTime - startTime; + System.out.println(elapsedTime/1000000 + " ms"); + } + + /** + * invokes the load Test + * A few sample invocations are as below + * TestProxyUsers ip 128 256 + * TestProxyUsers range 1.2.3.0/25 256 + * TestProxyUsers ip 4 8 + * TestProxyUsers range 1.2.3.0/30 8 + * @param args + */ + public static void main (String[] args){ + String ipValues = null; + + if (args.length != 3 || (!args[0].equals("ip") && !args[0].equals("range"))) { + System.out.println("Invalid invocation. The right syntax is ip/range "); + } + else { + if (args[0].equals("ip")){ + int numberOfIps = Integer.parseInt(args[1]); + StringBuilder sb = new StringBuilder(); + for (int i=0; i < numberOfIps; i++){ + sb.append("1.2.3."+ i + ","); + } + ipValues = sb.toString(); + } + else if (args[0].equals("range")){ + ipValues = args[1]; + } + + int testRange = Integer.parseInt(args[2]); + + loadTest(ipValues, testRange); + } + } + } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestMachineList.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestMachineList.java new file mode 100644 index 00000000000..2aa61fed357 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestMachineList.java @@ -0,0 +1,291 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Collection; + +import org.junit.Test; +import org.mockito.Mockito; + +public class TestMachineList { + private static String IP_LIST = "10.119.103.110,10.119.103.112,10.119.103.114"; + private static String IP_LIST_SPACES = + " 10.119.103.110 , 10.119.103.112,10.119.103.114 ,10.119.103.110, "; + private static String CIDR_LIST = "10.222.0.0/16,10.241.23.0/24"; + private static String CIDR_LIST1 = "10.222.0.0/16"; + private static String CIDR_LIST2 = "10.241.23.0/24"; + private static String INVALID_CIDR = "10.241/24"; + private static String IP_CIDR_LIST = + "10.222.0.0/16,10.119.103.110,10.119.103.112,10.119.103.114,10.241.23.0/24"; + private static String HOST_LIST = "host1,host4"; + private static String HOSTNAME_IP_CIDR_LIST = + "host1,10.222.0.0/16,10.119.103.110,10.119.103.112,10.119.103.114,10.241.23.0/24,host4,"; + + @Test + public void testWildCard() { + //create MachineList with a list of of IPs + MachineList ml = new MachineList("*"); + + //test for inclusion with any IP + assertTrue(ml.includes("10.119.103.112")); + assertTrue(ml.includes("1.2.3.4")); + } + + @Test + public void testIPList() { + //create MachineList with a list of of IPs + MachineList ml = new MachineList(IP_LIST); + + //test for inclusion with an known IP + assertTrue(ml.includes("10.119.103.112")); + + //test for exclusion with an unknown IP + assertFalse(ml.includes("10.119.103.111")); + } + + @Test + public void testIPListSpaces() { + //create MachineList with a ip string which has duplicate ip and spaces + MachineList ml = new MachineList(IP_LIST_SPACES); + + //test for inclusion with an known IP + assertTrue(ml.includes("10.119.103.112")); + + //test for exclusion with an unknown IP + assertFalse(ml.includes("10.119.103.111")); + } + + @Test + public void testStaticIPHostNameList()throws UnknownHostException { + //create MachineList with a list of of Hostnames + InetAddress addressHost1 = InetAddress.getByName("1.2.3.1"); + InetAddress addressHost4 = InetAddress.getByName("1.2.3.4"); + + MachineList.InetAddressFactory addressFactory = + Mockito.mock(MachineList.InetAddressFactory.class); + Mockito.when(addressFactory.getByName("host1")).thenReturn(addressHost1); + Mockito.when(addressFactory.getByName("host4")).thenReturn(addressHost4); + + MachineList ml = new MachineList( + StringUtils.getTrimmedStringCollection(HOST_LIST), addressFactory); + + //test for inclusion with an known IP + assertTrue(ml.includes("1.2.3.4")); + + //test for exclusion with an unknown IP + assertFalse(ml.includes("1.2.3.5")); + } + + @Test + public void testHostNames() throws UnknownHostException { + //create MachineList with a list of of Hostnames + InetAddress addressHost1 = InetAddress.getByName("1.2.3.1"); + InetAddress addressHost4 = InetAddress.getByName("1.2.3.4"); + InetAddress addressMockHost4 = Mockito.mock(InetAddress.class); + Mockito.when(addressMockHost4.getCanonicalHostName()).thenReturn("differentName"); + + InetAddress addressMockHost5 = Mockito.mock(InetAddress.class); + Mockito.when(addressMockHost5.getCanonicalHostName()).thenReturn("host5"); + + MachineList.InetAddressFactory addressFactory = + Mockito.mock(MachineList.InetAddressFactory.class); + Mockito.when(addressFactory.getByName("1.2.3.4")).thenReturn(addressMockHost4); + Mockito.when(addressFactory.getByName("1.2.3.5")).thenReturn(addressMockHost5); + Mockito.when(addressFactory.getByName("host1")).thenReturn(addressHost1); + Mockito.when(addressFactory.getByName("host4")).thenReturn(addressHost4); + + MachineList ml = new MachineList( + StringUtils.getTrimmedStringCollection(HOST_LIST), addressFactory ); + + //test for inclusion with an known IP + assertTrue(ml.includes("1.2.3.4")); + + //test for exclusion with an unknown IP + assertFalse(ml.includes("1.2.3.5")); + } + + @Test + public void testHostNamesReverserIpMatch() throws UnknownHostException { + //create MachineList with a list of of Hostnames + InetAddress addressHost1 = InetAddress.getByName("1.2.3.1"); + InetAddress addressHost4 = InetAddress.getByName("1.2.3.4"); + InetAddress addressMockHost4 = Mockito.mock(InetAddress.class); + Mockito.when(addressMockHost4.getCanonicalHostName()).thenReturn("host4"); + + InetAddress addressMockHost5 = Mockito.mock(InetAddress.class); + Mockito.when(addressMockHost5.getCanonicalHostName()).thenReturn("host5"); + + MachineList.InetAddressFactory addressFactory = + Mockito.mock(MachineList.InetAddressFactory.class); + Mockito.when(addressFactory.getByName("1.2.3.4")).thenReturn(addressMockHost4); + Mockito.when(addressFactory.getByName("1.2.3.5")).thenReturn(addressMockHost5); + Mockito.when(addressFactory.getByName("host1")).thenReturn(addressHost1); + Mockito.when(addressFactory.getByName("host4")).thenReturn(addressHost4); + + MachineList ml = new MachineList( + StringUtils.getTrimmedStringCollection(HOST_LIST), addressFactory ); + + //test for inclusion with an known IP + assertTrue(ml.includes("1.2.3.4")); + + //test for exclusion with an unknown IP + assertFalse(ml.includes("1.2.3.5")); + } + + @Test + public void testCIDRs() { + //create MachineList with a list of of ip ranges specified in CIDR format + MachineList ml = new MachineList(CIDR_LIST); + + //test for inclusion/exclusion + assertFalse(ml.includes("10.221.255.255")); + assertTrue(ml.includes("10.222.0.0")); + assertTrue(ml.includes("10.222.0.1")); + assertTrue(ml.includes("10.222.0.255")); + assertTrue(ml.includes("10.222.255.0")); + assertTrue(ml.includes("10.222.255.254")); + assertTrue(ml.includes("10.222.255.255")); + assertFalse(ml.includes("10.223.0.0")); + + assertTrue(ml.includes("10.241.23.0")); + assertTrue(ml.includes("10.241.23.1")); + assertTrue(ml.includes("10.241.23.254")); + assertTrue(ml.includes("10.241.23.255")); + + //test for exclusion with an unknown IP + assertFalse(ml.includes("10.119.103.111")); + + } + + @Test + public void testCIDRWith16bitmask() { + //create MachineList with a list of of ip ranges specified in CIDR format + MachineList ml = new MachineList(CIDR_LIST1); + + //test for inclusion/exclusion + assertFalse(ml.includes("10.221.255.255")); + assertTrue(ml.includes("10.222.0.0")); + assertTrue(ml.includes("10.222.0.1")); + assertTrue(ml.includes("10.222.0.255")); + assertTrue(ml.includes("10.222.255.0")); + assertTrue(ml.includes("10.222.255.254")); + assertTrue(ml.includes("10.222.255.255")); + assertFalse(ml.includes("10.223.0.0")); + + //test for exclusion with an unknown IP + assertFalse(ml.includes("10.119.103.111")); + } + + @Test + public void testCIDRWith8BitMask() { + //create MachineList with a list of of ip ranges specified in CIDR format + MachineList ml = new MachineList(CIDR_LIST2); + + //test for inclusion/exclusion + assertFalse(ml.includes("10.241.22.255")); + assertTrue(ml.includes("10.241.23.0")); + assertTrue(ml.includes("10.241.23.1")); + assertTrue(ml.includes("10.241.23.254")); + assertTrue(ml.includes("10.241.23.255")); + assertFalse(ml.includes("10.241.24.0")); + + //test for exclusion with an unknown IP + assertFalse(ml.includes("10.119.103.111")); + } + + //test invalid cidr + @Test + public void testInvalidCIDR() { + //create MachineList with an Invalid CIDR + try { + new MachineList(INVALID_CIDR); + fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + //expected Exception + } catch (Throwable t) { + fail ("Expected only IllegalArgumentException"); + } + } + // + @Test + public void testIPandCIDRs() { + //create MachineList with a list of of ip ranges and ip addresses + MachineList ml = new MachineList(IP_CIDR_LIST); + + //test for inclusion with an known IP + assertTrue(ml.includes("10.119.103.112")); + + //test for exclusion with an unknown IP + assertFalse(ml.includes("10.119.103.111")); + + //CIDR Ranges + assertFalse(ml.includes("10.221.255.255")); + assertTrue(ml.includes("10.222.0.0")); + assertTrue(ml.includes("10.222.255.255")); + assertFalse(ml.includes("10.223.0.0")); + + assertFalse(ml.includes("10.241.22.255")); + assertTrue(ml.includes("10.241.23.0")); + assertTrue(ml.includes("10.241.23.255")); + assertFalse(ml.includes("10.241.24.0")); + } + + @Test + public void testHostNameIPandCIDRs() { + //create MachineList with a mix of ip addresses , hostnames and ip ranges + MachineList ml = new MachineList(HOSTNAME_IP_CIDR_LIST); + + //test for inclusion with an known IP + assertTrue(ml.includes("10.119.103.112")); + + //test for exclusion with an unknown IP + assertFalse(ml.includes("10.119.103.111")); + + //CIDR Ranges + assertFalse(ml.includes("10.221.255.255")); + assertTrue(ml.includes("10.222.0.0")); + assertTrue(ml.includes("10.222.255.255")); + assertFalse(ml.includes("10.223.0.0")); + + assertFalse(ml.includes("10.241.22.255")); + assertTrue(ml.includes("10.241.23.0")); + assertTrue(ml.includes("10.241.23.255")); + assertFalse(ml.includes("10.241.24.0")); + } + + @Test + public void testGetCollection() { + //create MachineList with a mix of ip addresses , hostnames and ip ranges + MachineList ml = new MachineList(HOSTNAME_IP_CIDR_LIST); + + Collection col = ml.getCollection(); + //test getCollectionton to return the full collection + assertEquals(7,ml.getCollection().size()); + + for (String item:StringUtils.getTrimmedStringCollection(HOSTNAME_IP_CIDR_LIST)) { + assertTrue(col.contains(item)); + } + } +} From 5f880f79d275c74475836a1932be6f6f2daa1407 Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Fri, 27 Jun 2014 12:00:55 +0000 Subject: [PATCH 058/112] HADOOP-10701. NFS should not validate the access premission only based on the user's primary group. Contributed by Harsh J. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1606042 13f79535-47bb-0310-9956-ffa450edef68 --- .../dev-support/findbugsExcludeFile.xml | 28 ++++++++ hadoop-common-project/hadoop-nfs/pom.xml | 12 ++++ .../oncrpc/security/CredentialsSys.java | 6 +- .../oncrpc/security/SecurityHandler.java | 5 ++ .../oncrpc/security/SysSecurityHandler.java | 5 ++ .../hadoop/hdfs/nfs/nfs3/Nfs3Utils.java | 12 +++- .../hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java | 6 +- .../hadoop/hdfs/nfs/nfs3/TestNfs3Utils.java | 72 +++++++++++++++++++ hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + 9 files changed, 144 insertions(+), 5 deletions(-) create mode 100644 hadoop-common-project/hadoop-nfs/dev-support/findbugsExcludeFile.xml create mode 100644 hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestNfs3Utils.java diff --git a/hadoop-common-project/hadoop-nfs/dev-support/findbugsExcludeFile.xml b/hadoop-common-project/hadoop-nfs/dev-support/findbugsExcludeFile.xml new file mode 100644 index 00000000000..d76cc77a32a --- /dev/null +++ b/hadoop-common-project/hadoop-nfs/dev-support/findbugsExcludeFile.xml @@ -0,0 +1,28 @@ + + + + + + + + + diff --git a/hadoop-common-project/hadoop-nfs/pom.xml b/hadoop-common-project/hadoop-nfs/pom.xml index 56250660bcd..e30a482edf6 100644 --- a/hadoop-common-project/hadoop-nfs/pom.xml +++ b/hadoop-common-project/hadoop-nfs/pom.xml @@ -93,6 +93,18 @@ + + + + org.codehaus.mojo + findbugs-maven-plugin + + ${basedir}/dev-support/findbugsExcludeFile.xml + + + + + diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/CredentialsSys.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/CredentialsSys.java index 9ba12b8990e..997ce35c60d 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/CredentialsSys.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/CredentialsSys.java @@ -58,6 +58,10 @@ public int getUID() { return mUID; } + public int[] getAuxGIDs() { + return mAuxGIDs; + } + public void setGID(int gid) { this.mGID = gid; } @@ -65,7 +69,7 @@ public void setGID(int gid) { public void setUID(int uid) { this.mUID = uid; } - + public void setStamp(int stamp) { this.mStamp = stamp; } diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/SecurityHandler.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/SecurityHandler.java index 40004d0e786..063082e4c4d 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/SecurityHandler.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/SecurityHandler.java @@ -60,4 +60,9 @@ public int getUid() { public int getGid() { throw new UnsupportedOperationException(); } + + /** Used by AUTH_SYS */ + public int[] getAuxGids() { + throw new UnsupportedOperationException(); + } } diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/SysSecurityHandler.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/SysSecurityHandler.java index 196d3d8cbba..74237764c5f 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/SysSecurityHandler.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/SysSecurityHandler.java @@ -56,4 +56,9 @@ public int getUid() { public int getGid() { return mCredentialsSys.getGID(); } + + @Override + public int[] getAuxGids() { + return mCredentialsSys.getAuxGIDs(); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3Utils.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3Utils.java index b351fda68f9..5c30f16bc97 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3Utils.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3Utils.java @@ -160,7 +160,7 @@ public static int getAccessRights(int mode, int type) { } public static int getAccessRightsForUserGroup(int uid, int gid, - Nfs3FileAttributes attr) { + int[] auxGids, Nfs3FileAttributes attr) { int mode = attr.getMode(); if (uid == attr.getUid()) { return getAccessRights(mode >> 6, attr.getType()); @@ -168,6 +168,14 @@ public static int getAccessRightsForUserGroup(int uid, int gid, if (gid == attr.getGid()) { return getAccessRights(mode >> 3, attr.getType()); } + // Check for membership in auxiliary groups + if (auxGids != null) { + for (int auxGid : auxGids) { + if (attr.getGid() == auxGid) { + return getAccessRights(mode >> 3, attr.getType()); + } + } + } return getAccessRights(mode, attr.getType()); } @@ -191,4 +199,4 @@ public static byte[] longToByte(long v) { data[7] = (byte) (v >>> 0); return data; } -} \ No newline at end of file +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java index 446e722a213..96a4c494987 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java @@ -504,7 +504,8 @@ public ACCESS3Response access(XDR xdr, RpcInfo info) { return new ACCESS3Response(Nfs3Status.NFS3ERR_STALE); } int access = Nfs3Utils.getAccessRightsForUserGroup( - securityHandler.getUid(), securityHandler.getGid(), attrs); + securityHandler.getUid(), securityHandler.getGid(), + securityHandler.getAuxGids(), attrs); return new ACCESS3Response(Nfs3Status.NFS3_OK, attrs, access); } catch (RemoteException r) { @@ -659,7 +660,8 @@ READ3Response read(XDR xdr, SecurityHandler securityHandler, return new READ3Response(Nfs3Status.NFS3ERR_NOENT); } int access = Nfs3Utils.getAccessRightsForUserGroup( - securityHandler.getUid(), securityHandler.getGid(), attrs); + securityHandler.getUid(), securityHandler.getGid(), + securityHandler.getAuxGids(), attrs); if ((access & Nfs3Constant.ACCESS3_READ) != 0) { eof = offset < attrs.getSize() ? false : true; return new READ3Response(Nfs3Status.NFS3_OK, attrs, 0, eof, diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestNfs3Utils.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestNfs3Utils.java new file mode 100644 index 00000000000..b5f0cd4c539 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestNfs3Utils.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.nfs.nfs3; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import org.junit.Test; + +import java.io.IOException; + +import org.apache.hadoop.nfs.NfsFileType; +import org.apache.hadoop.nfs.nfs3.Nfs3FileAttributes; + +import org.mockito.Mockito; + +public class TestNfs3Utils { + @Test + public void testGetAccessRightsForUserGroup() throws IOException { + Nfs3FileAttributes attr = Mockito.mock(Nfs3FileAttributes.class); + Mockito.when(attr.getUid()).thenReturn(2); + Mockito.when(attr.getGid()).thenReturn(3); + Mockito.when(attr.getMode()).thenReturn(448); // 700 + Mockito.when(attr.getType()).thenReturn(NfsFileType.NFSREG.toValue()); + assertEquals("No access should be allowed as UID does not match attribute over mode 700", + 0, Nfs3Utils.getAccessRightsForUserGroup(3, 3, null, attr)); + Mockito.when(attr.getUid()).thenReturn(2); + Mockito.when(attr.getGid()).thenReturn(3); + Mockito.when(attr.getMode()).thenReturn(56); // 070 + Mockito.when(attr.getType()).thenReturn(NfsFileType.NFSREG.toValue()); + assertEquals("No access should be allowed as GID does not match attribute over mode 070", + 0, Nfs3Utils.getAccessRightsForUserGroup(2, 4, null, attr)); + Mockito.when(attr.getUid()).thenReturn(2); + Mockito.when(attr.getGid()).thenReturn(3); + Mockito.when(attr.getMode()).thenReturn(7); // 007 + Mockito.when(attr.getType()).thenReturn(NfsFileType.NFSREG.toValue()); + assertEquals("Access should be allowed as mode is 007 and UID/GID do not match", + 61 /* RWX */, Nfs3Utils.getAccessRightsForUserGroup(1, 4, new int[] {5, 6}, attr)); + Mockito.when(attr.getUid()).thenReturn(2); + Mockito.when(attr.getGid()).thenReturn(10); + Mockito.when(attr.getMode()).thenReturn(288); // 440 + Mockito.when(attr.getType()).thenReturn(NfsFileType.NFSREG.toValue()); + assertEquals("Access should be allowed as mode is 440 and Aux GID does match", + 1 /* R */, Nfs3Utils.getAccessRightsForUserGroup(3, 4, new int[] {5, 16, 10}, attr)); + Mockito.when(attr.getUid()).thenReturn(2); + Mockito.when(attr.getGid()).thenReturn(10); + Mockito.when(attr.getMode()).thenReturn(448); // 700 + Mockito.when(attr.getType()).thenReturn(NfsFileType.NFSDIR.toValue()); + assertEquals("Access should be allowed for dir as mode is 700 and UID does match", + 31 /* Lookup */, Nfs3Utils.getAccessRightsForUserGroup(2, 4, new int[] {5, 16, 10}, attr)); + assertEquals("No access should be allowed for dir as mode is 700 even though GID does match", + 0, Nfs3Utils.getAccessRightsForUserGroup(3, 10, new int[] {5, 16, 4}, attr)); + assertEquals("No access should be allowed for dir as mode is 700 even though AuxGID does match", + 0, Nfs3Utils.getAccessRightsForUserGroup(3, 20, new int[] {5, 10}, attr)); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index e3ca235b335..a4220b09627 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -725,6 +725,9 @@ Release 2.5.0 - UNRELEASED HDFS-6475. WebHdfs clients fail without retry because incorrect handling of StandbyException. (Yongjun Zhang via atm) + HADOOP-10701. NFS should not validate the access premission only based on + the user's primary group (Harsh J via atm) + BREAKDOWN OF HDFS-2006 SUBTASKS AND RELATED JIRAS HDFS-6299. Protobuf for XAttr and client-side implementation. (Yi Liu via umamahesh) From f911f5495ba73ce62b28b9492c03248e07c7e7d1 Mon Sep 17 00:00:00 2001 From: Karthik Kambatla Date: Fri, 27 Jun 2014 18:09:41 +0000 Subject: [PATCH 059/112] YARN-2204. Addendum patch. TestAMRestart#testAMRestartWithExistingContainers assumes CapacityScheduler. (Robert Kanter via kasha) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1606168 13f79535-47bb-0310-9956-ffa450edef68 --- .../applicationsmanager/TestAMRestart.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java index ec4ed0ea953..4145b6411d8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java @@ -45,6 +45,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; @@ -57,8 +58,6 @@ public class TestAMRestart { public void testAMRestartWithExistingContainers() throws Exception { YarnConfiguration conf = new YarnConfiguration(); conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2); - conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, - ResourceScheduler.class); MockRM rm1 = new MockRM(conf); rm1.start(); @@ -125,9 +124,9 @@ public void testAMRestartWithExistingContainers() throws Exception { ContainerId.newInstance(am1.getApplicationAttemptId(), 6); nm1.nodeHeartbeat(true); SchedulerApplicationAttempt schedulerAttempt = - ((CapacityScheduler) rm1.getResourceScheduler()) + ((AbstractYarnScheduler) rm1.getResourceScheduler()) .getCurrentAttemptForContainer(containerId6); - while (schedulerAttempt.getReservedContainers().size() == 0) { + while (schedulerAttempt.getReservedContainers().isEmpty()) { System.out.println("Waiting for container " + containerId6 + " to be reserved."); nm1.nodeHeartbeat(true); @@ -221,7 +220,7 @@ public void testAMRestartWithExistingContainers() throws Exception { // record the scheduler attempt for testing. SchedulerApplicationAttempt schedulerNewAttempt = - ((CapacityScheduler) rm1.getResourceScheduler()) + ((AbstractYarnScheduler) rm1.getResourceScheduler()) .getCurrentAttemptForContainer(containerId2); // finish this application MockRM.finishAMAndVerifyAppState(app1, rm1, nm1, am2); From bbbbd270c7ff0fba55ecd863104ced4c27a8478b Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Fri, 27 Jun 2014 18:43:42 +0000 Subject: [PATCH 060/112] HADOOP-10649. Allow overriding the default ACL for service authorization (Contributed by Benoy Antony) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1606179 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../hadoop/fs/CommonConfigurationKeys.java | 3 + .../ServiceAuthorizationManager.java | 6 +- .../src/site/apt/ServiceLevelAuth.apt.vm | 8 ++- .../authorize/TestServiceAuthorization.java | 67 +++++++++++++++++++ 5 files changed, 84 insertions(+), 3 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestServiceAuthorization.java diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 3603d7ed6d2..27d05ce624b 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -483,6 +483,9 @@ Release 2.5.0 - UNRELEASED HADOOP-10565. Support IP ranges (CIDR) in proxyuser.hosts. (Benoy Antony via Arpit Agarwal) + HADOOP-10649. Allow overriding the default ACL for service authorization + (Benoy Antony via Arpit Agarwal) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java index a1932c07c38..3345e3c93d5 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java @@ -131,6 +131,9 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic { * Service Authorization */ public static final String + HADOOP_SECURITY_SERVICE_AUTHORIZATION_DEFAULT_ACL = + "security.service.authorization.default.acl"; + public static final String HADOOP_SECURITY_SERVICE_AUTHORIZATION_REFRESH_POLICY = "security.refresh.policy.protocol.acl"; public static final String diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java index 29d4a6ac47b..d12ab79c798 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java @@ -131,6 +131,10 @@ public void refreshWithLoadedConfiguration(Configuration conf, PolicyProvider provider) { final Map, AccessControlList> newAcls = new IdentityHashMap, AccessControlList>(); + + String defaultAcl = conf.get( + CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_AUTHORIZATION_DEFAULT_ACL, + AccessControlList.WILDCARD_ACL_VALUE); // Parse the config file Service[] services = provider.getServices(); @@ -139,7 +143,7 @@ public void refreshWithLoadedConfiguration(Configuration conf, AccessControlList acl = new AccessControlList( conf.get(service.getServiceKey(), - AccessControlList.WILDCARD_ACL_VALUE) + defaultAcl) ); newAcls.put(service.getProtocol(), acl); } diff --git a/hadoop-common-project/hadoop-common/src/site/apt/ServiceLevelAuth.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/ServiceLevelAuth.apt.vm index 258819e1107..6a11f3f643d 100644 --- a/hadoop-common-project/hadoop-common/src/site/apt/ServiceLevelAuth.apt.vm +++ b/hadoop-common-project/hadoop-common/src/site/apt/ServiceLevelAuth.apt.vm @@ -100,11 +100,15 @@ security.ha.service.protocol.acl | ACL for HAService protocol used by HAAdm Example: <<>>. Add a blank at the beginning of the line if only a list of groups is to - be provided, equivalently a comman-separated list of users followed by + be provided, equivalently a comma-separated list of users followed by a space or nothing implies only a set of given users. A special value of <<<*>>> implies that all users are allowed to access the - service. + service. + + If access control list is not defined for a service, the value of + <<>> is applied. If + <<>> is not defined, <<<*>>> is applied. ** Refreshing Service Level Authorization Configuration diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestServiceAuthorization.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestServiceAuthorization.java new file mode 100644 index 00000000000..f6cf8bce2e9 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestServiceAuthorization.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.security.authorize; + +import static org.junit.Assert.assertEquals; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.ipc.TestRPC.TestProtocol; +import org.junit.Test; + +public class TestServiceAuthorization { + + private static final String ACL_CONFIG = "test.protocol.acl"; + private static final String ACL_CONFIG1 = "test.protocol1.acl"; + + public interface TestProtocol1 extends TestProtocol {}; + + private static class TestPolicyProvider extends PolicyProvider { + + @Override + public Service[] getServices() { + return new Service[] { new Service(ACL_CONFIG, TestProtocol.class), + new Service(ACL_CONFIG1, TestProtocol1.class), + }; + } + } + + @Test + public void testDefaultAcl() { + ServiceAuthorizationManager serviceAuthorizationManager = + new ServiceAuthorizationManager(); + Configuration conf = new Configuration (); + //test without setting a default acl + conf.set(ACL_CONFIG, "user1 group1"); + serviceAuthorizationManager.refresh(conf, new TestPolicyProvider()); + AccessControlList acl = serviceAuthorizationManager.getProtocolsAcls(TestProtocol.class); + assertEquals("user1 group1", acl.getAclString()); + acl = serviceAuthorizationManager.getProtocolsAcls(TestProtocol1.class); + assertEquals(AccessControlList.WILDCARD_ACL_VALUE, acl.getAclString()); + + //test with a default acl + conf.set( + CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_AUTHORIZATION_DEFAULT_ACL, + "user2 group2"); + serviceAuthorizationManager.refresh(conf, new TestPolicyProvider()); + acl = serviceAuthorizationManager.getProtocolsAcls(TestProtocol.class); + assertEquals("user1 group1", acl.getAclString()); + acl = serviceAuthorizationManager.getProtocolsAcls(TestProtocol1.class); + assertEquals("user2 group2", acl.getAclString()); + } +} From 73927ba6957181a121e60b0732310ce4185f96e7 Mon Sep 17 00:00:00 2001 From: Jason Darrell Lowe Date: Fri, 27 Jun 2014 23:11:12 +0000 Subject: [PATCH 061/112] YARN-2104. Scheduler queue filter failed to work because index of queue column changed. Contributed by Wangda Tan git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1606265 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../server/resourcemanager/webapp/CapacitySchedulerPage.java | 2 +- .../server/resourcemanager/webapp/DefaultSchedulerPage.java | 2 +- .../yarn/server/resourcemanager/webapp/FairSchedulerPage.java | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 6d17d6ae447..58e394bf538 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -296,6 +296,9 @@ Release 2.5.0 - UNRELEASED YARN-2163. WebUI: Order of AppId in apps table should be consistent with ApplicationId.compareTo(). (Wangda Tan via raviprak) + YARN-2104. Scheduler queue filter failed to work because index of queue + column changed. (Wangda Tan via jlowe) + Release 2.4.1 - 2014-06-23 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java index 0f0ed5052cc..a53ad9847e8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java @@ -262,7 +262,7 @@ public void render(Block html) { " var q = $('.q', data.rslt.obj).first().text();", " if (q == 'root') q = '';", " else q = '^' + q.substr(q.lastIndexOf('.') + 1) + '$';", - " $('#apps').dataTable().fnFilter(q, 3, true);", + " $('#apps').dataTable().fnFilter(q, 4, true);", " });", " $('#cs').show();", "});")._(). diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DefaultSchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DefaultSchedulerPage.java index 1bbb993ac80..71fbbf5db39 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DefaultSchedulerPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DefaultSchedulerPage.java @@ -137,7 +137,7 @@ public void render(Block html) { " $('#cs').bind('select_node.jstree', function(e, data) {", " var q = $('.q', data.rslt.obj).first().text();", " if (q == 'root') q = '';", - " $('#apps').dataTable().fnFilter(q, 3);", + " $('#apps').dataTable().fnFilter(q, 4);", " });", " $('#cs').show();", "});")._(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java index dfa7e25c4c5..aca3e448485 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java @@ -210,7 +210,7 @@ public void render(Block html) { " var q = $('.q', data.rslt.obj).first().text();", " if (q == 'root') q = '';", " else q = '^' + q.substr(q.lastIndexOf('.') + 1) + '$';", - " $('#apps').dataTable().fnFilter(q, 3, true);", + " $('#apps').dataTable().fnFilter(q, 4, true);", " });", " $('#cs').show();", "});")._(). From 739b135dccedfc1faceef1fb1b8574d917a5f75d Mon Sep 17 00:00:00 2001 From: Jason Darrell Lowe Date: Fri, 27 Jun 2014 23:22:35 +0000 Subject: [PATCH 062/112] HADOOP-10739. Renaming a file into a directory containing the same filename results in a confusing I/O error. Contributed by chang li git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1606267 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../apache/hadoop/fs/shell/MoveCommands.java | 3 + .../org/apache/hadoop/fs/shell/TestMove.java | 123 ++++++++++++++++++ 3 files changed, 129 insertions(+) create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestMove.java diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 27d05ce624b..4a3f548ab5e 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -632,6 +632,9 @@ Release 2.5.0 - UNRELEASED HADOOP-9705. FsShell cp -p does not preserve directory attibutes. (Akira AJISAKA via cnauroth) + HADOOP-10739. Renaming a file into a directory containing the same + filename results in a confusing I/O error (chang li via jlowe) + BREAKDOWN OF HADOOP-10514 SUBTASKS AND RELATED JIRAS HADOOP-10520. Extended attributes definition and FileSystem APIs for diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java index 4e347efbdc8..1c7316ae65c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java @@ -104,6 +104,9 @@ protected void processPath(PathData src, PathData target) throws IOException { throw new PathIOException(src.toString(), "Does not match target filesystem"); } + if (target.exists) { + throw new PathExistsException(target.toString()); + } if (!target.fs.rename(src.path, target.path)) { // we have no way to know the actual error... throw new PathIOException(src.toString()); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestMove.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestMove.java new file mode 100644 index 00000000000..6599edf786b --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestMove.java @@ -0,0 +1,123 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.shell; + +import static org.junit.Assert.*; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.*; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FilterFileSystem; +import org.apache.hadoop.fs.PathExistsException; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestMove { + static Configuration conf; + static FileSystem mockFs; + + @BeforeClass + public static void setup() throws IOException, URISyntaxException { + mockFs = mock(FileSystem.class); + conf = new Configuration(); + conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class); + } + + @Before + public void resetMock() throws IOException { + reset(mockFs); + } + + @Test + public void testMoveTargetExistsWithoutExplicitRename() throws Exception { + Path srcPath = new Path("mockfs:/file"); + Path targetPath = new Path("mockfs:/fold0"); + Path dupPath = new Path("mockfs:/fold0/file"); + Path srcPath2 = new Path("mockfs://user/file"); + Path targetPath2 = new Path("mockfs://user/fold0"); + Path dupPath2 = new Path("mockfs://user/fold0/file"); + InstrumentedRenameCommand cmd; + String[] cmdargs = new String[]{"mockfs:/file", "mockfs:/fold0"}; + FileStatus src_fileStat, target_fileStat, dup_fileStat; + URI myuri; + + src_fileStat = mock(FileStatus.class); + target_fileStat = mock(FileStatus.class); + dup_fileStat = mock(FileStatus.class); + myuri = new URI("mockfs://user"); + + when(src_fileStat.isDirectory()).thenReturn(false); + when(target_fileStat.isDirectory()).thenReturn(true); + when(dup_fileStat.isDirectory()).thenReturn(false); + when(src_fileStat.getPath()).thenReturn(srcPath2); + when(target_fileStat.getPath()).thenReturn(targetPath2); + when(dup_fileStat.getPath()).thenReturn(dupPath2); + when(mockFs.getFileStatus(eq(srcPath))).thenReturn(src_fileStat); + when(mockFs.getFileStatus(eq(targetPath))).thenReturn(target_fileStat); + when(mockFs.getFileStatus(eq(dupPath))).thenReturn(dup_fileStat); + when(mockFs.getFileStatus(eq(srcPath2))).thenReturn(src_fileStat); + when(mockFs.getFileStatus(eq(targetPath2))).thenReturn(target_fileStat); + when(mockFs.getFileStatus(eq(dupPath2))).thenReturn(dup_fileStat); + when(mockFs.getUri()).thenReturn(myuri); + + cmd = new InstrumentedRenameCommand(); + cmd.setConf(conf); + cmd.setOverwrite(true); + cmd.run(cmdargs); + + // make sure command failed with the proper exception + assertTrue("Rename should have failed with path exists exception", + cmd.error instanceof PathExistsException); + } + + static class MockFileSystem extends FilterFileSystem { + Configuration conf; + MockFileSystem() { + super(mockFs); + } + @Override + public void initialize(URI uri, Configuration conf) { + this.conf = conf; + } + @Override + public Path makeQualified(Path path) { + return path; + } + @Override + public Configuration getConf() { + return conf; + } + } + + private static class InstrumentedRenameCommand extends MoveCommands.Rename { + private Exception error = null; + @Override + public void displayError(Exception e) { + error = e; + } + } +} From 55a0aa0bade40bc4c95b9fedad165046b313c4b5 Mon Sep 17 00:00:00 2001 From: Zhijie Shen Date: Sat, 28 Jun 2014 03:30:44 +0000 Subject: [PATCH 063/112] YARN-2201. Made TestRMWebServicesAppsModification be independent of the changes on yarn-default.xml. Contributed by Varun Vasudev. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1606285 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../TestRMWebServicesAppsModification.java | 27 ++++++++++++------- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 58e394bf538..ef0111f09de 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -299,6 +299,9 @@ Release 2.5.0 - UNRELEASED YARN-2104. Scheduler queue filter failed to work because index of queue column changed. (Wangda Tan via jlowe) + YARN-2201. Made TestRMWebServicesAppsModification be independent of the + changes on yarn-default.xml. (Varun Vasudev via zjshen) + Release 2.4.1 - 2014-06-23 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java index 7cbf125ec57..76281d47b80 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java @@ -20,6 +20,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; import java.io.IOException; import java.io.StringReader; @@ -49,6 +50,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppState; @@ -93,6 +95,8 @@ public class TestRMWebServicesAppsModification extends JerseyTest { private Injector injector; private String webserviceUserName = "testuser"; + private boolean setAuthFilter = false; + public class GuiceServletConfig extends GuiceServletContextListener { @Override @@ -131,7 +135,6 @@ protected Properties getConfiguration(String configPrefix, private class TestServletModule extends ServletModule { public Configuration conf = new Configuration(); - boolean setAuthFilter = false; @Override protected void configureServlets() { @@ -157,6 +160,7 @@ private Injector getNoAuthInjector() { return Guice.createInjector(new TestServletModule() { @Override protected void configureServlets() { + setAuthFilter = false; super.configureServlets(); } }); @@ -204,8 +208,8 @@ public TestRMWebServicesAppsModification(int run) { } } - private boolean isAuthorizationEnabled() { - return rm.getConfig().getBoolean(YarnConfiguration.YARN_ACL_ENABLE, false); + private boolean isAuthenticationEnabled() { + return setAuthFilter; } private WebResource constructWebResource(WebResource r, String... paths) { @@ -213,7 +217,7 @@ private WebResource constructWebResource(WebResource r, String... paths) { for (String path : paths) { rt = rt.path(path); } - if (isAuthorizationEnabled()) { + if (isAuthenticationEnabled()) { rt = rt.queryParam("user.name", webserviceUserName); } return rt; @@ -280,7 +284,7 @@ public void testSingleAppKill() throws Exception { "state").entity(entity, contentType).accept(mediaType) .put(ClientResponse.class); - if (!isAuthorizationEnabled()) { + if (!isAuthenticationEnabled()) { assertEquals(Status.UNAUTHORIZED, response.getClientResponseStatus()); continue; } @@ -295,7 +299,7 @@ public void testSingleAppKill() throws Exception { response.getHeaders().getFirst(HttpHeaders.LOCATION); Client c = Client.create(); WebResource tmp = c.resource(locationHeaderValue); - if (isAuthorizationEnabled()) { + if (isAuthenticationEnabled()) { tmp = tmp.queryParam("user.name", webserviceUserName); } response = tmp.get(ClientResponse.class); @@ -361,7 +365,7 @@ public void testSingleAppKillInvalidState() throws Exception { .entity(entity, contentType).accept(mediaType) .put(ClientResponse.class); - if (!isAuthorizationEnabled()) { + if (!isAuthenticationEnabled()) { assertEquals(Status.UNAUTHORIZED, response.getClientResponseStatus()); continue; @@ -426,6 +430,11 @@ protected static void verifyAppStateXML(ClientResponse response, @Test(timeout = 30000) public void testSingleAppKillUnauthorized() throws Exception { + boolean isCapacityScheduler = + rm.getResourceScheduler() instanceof CapacityScheduler; + assumeTrue("Currently this test is only supported on CapacityScheduler", + isCapacityScheduler); + // default root queue allows anyone to have admin acl CapacitySchedulerConfiguration csconf = new CapacitySchedulerConfiguration(); @@ -452,7 +461,7 @@ public void testSingleAppKillUnauthorized() throws Exception { .constructWebResource("apps", app.getApplicationId().toString(), "state").accept(mediaType) .entity(info, MediaType.APPLICATION_XML).put(ClientResponse.class); - if (!isAuthorizationEnabled()) { + if (!isAuthenticationEnabled()) { assertEquals(Status.UNAUTHORIZED, response.getClientResponseStatus()); } else { assertEquals(Status.FORBIDDEN, response.getClientResponseStatus()); @@ -475,7 +484,7 @@ public void testSingleAppKillInvalidId() throws Exception { this.constructWebResource("apps", testAppId, "state") .accept(MediaType.APPLICATION_XML) .entity(info, MediaType.APPLICATION_XML).put(ClientResponse.class); - if (!isAuthorizationEnabled()) { + if (!isAuthenticationEnabled()) { assertEquals(Status.UNAUTHORIZED, response.getClientResponseStatus()); continue; } From 78cafe34e6ed218b409057aac09828bf1c9fae9c Mon Sep 17 00:00:00 2001 From: Uma Maheswara Rao G Date: Sat, 28 Jun 2014 11:40:02 +0000 Subject: [PATCH 064/112] HDFS-6556. Refine XAttr permissions. Contributed by Uma Maheswara Rao G. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1606320 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../main/java/org/apache/hadoop/fs/XAttr.java | 6 +- .../hdfs/server/namenode/FSNamesystem.java | 25 ++- .../namenode/XAttrPermissionFilter.java | 3 +- .../org/apache/hadoop/hdfs/TestDFSShell.java | 147 ++++++++++++------ 5 files changed, 121 insertions(+), 62 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index a4220b09627..9b5978bcd81 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -728,6 +728,8 @@ Release 2.5.0 - UNRELEASED HADOOP-10701. NFS should not validate the access premission only based on the user's primary group (Harsh J via atm) + HDFS-6556. Refine XAttr permissions (umamahesh) + BREAKDOWN OF HDFS-2006 SUBTASKS AND RELATED JIRAS HDFS-6299. Protobuf for XAttr and client-side implementation. (Yi Liu via umamahesh) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/XAttr.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/XAttr.java index 82272e22a31..99f629afdfe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/XAttr.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/XAttr.java @@ -30,11 +30,11 @@ * namespaces are defined: user, trusted, security and system. * 1) USER namespace attributes may be used by any user to store * arbitrary information. Access permissions in this namespace are - * defined by a file directory's permission bits. + * defined by a file directory's permission bits. For sticky directories, + * only the owner and privileged user can write attributes. *
    * 2) TRUSTED namespace attributes are only visible and accessible to - * privileged users (a file or directory's owner or the fs - * admin). This namespace is available from both user space + * privileged users. This namespace is available from both user space * (filesystem API) and fs kernel. *
    * 3) SYSTEM namespace attributes are used by the fs kernel to store diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 3cc19ed8d2f..a53964b8114 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -8196,10 +8196,7 @@ private void setXAttrInt(String src, XAttr xAttr, EnumSet flag, checkOperation(OperationCategory.WRITE); checkNameNodeSafeMode("Cannot set XAttr on " + src); src = FSDirectory.resolvePath(src, pathComponents, dir); - if (isPermissionEnabled) { - checkOwner(pc, src); - checkPathAccess(pc, src, FsAction.WRITE); - } + checkXAttrChangeAccess(src, xAttr, pc); List xAttrs = Lists.newArrayListWithCapacity(1); xAttrs.add(xAttr); dir.setXAttrs(src, xAttrs, flag); @@ -8319,10 +8316,7 @@ void removeXAttr(String src, XAttr xAttr) throws IOException { checkOperation(OperationCategory.WRITE); checkNameNodeSafeMode("Cannot remove XAttr entry on " + src); src = FSDirectory.resolvePath(src, pathComponents, dir); - if (isPermissionEnabled) { - checkOwner(pc, src); - checkPathAccess(pc, src, FsAction.WRITE); - } + checkXAttrChangeAccess(src, xAttr, pc); List xAttrs = Lists.newArrayListWithCapacity(1); xAttrs.add(xAttr); @@ -8341,6 +8335,21 @@ void removeXAttr(String src, XAttr xAttr) throws IOException { logAuditEvent(true, "removeXAttr", src, null, resultingStat); } + private void checkXAttrChangeAccess(String src, XAttr xAttr, + FSPermissionChecker pc) throws UnresolvedLinkException, + AccessControlException { + if (isPermissionEnabled && xAttr.getNameSpace() == XAttr.NameSpace.USER) { + final INode inode = dir.getINode(src); + if (inode.isDirectory() && inode.getFsPermission().getStickyBit()) { + if (!pc.isSuperUser()) { + checkOwner(pc, src); + } + } else { + checkPathAccess(pc, src, FsAction.WRITE); + } + } + } + /** * Default AuditLogger implementation; used when no access logger is * defined in the config file. It can also be explicitly listed in the diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java index 7fed362b9b2..47f29399e5a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java @@ -34,7 +34,8 @@ * USER - extended user attributes: these can be assigned to files and * directories to store arbitrary additional information. The access * permissions for user attributes are defined by the file permission - * bits. + * bits. For sticky directories, only the owner and privileged user can + * write attributes. *
    * TRUSTED - trusted extended attributes: these are visible/accessible * only to/by the super user. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java index 8788a2564b1..1c4549070e5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java @@ -2456,38 +2456,39 @@ private void doSetXattr(ByteArrayOutputStream out, FsShell fshell, } /** - * HDFS-6374 setXAttr should require the user to be the owner of the file - * or directory. - * - * Test to make sure that only the owner of a file or directory can set - * or remove the xattrs. - * - * As user1: - * Create a directory (/foo) as user1, chown it to user1 (and user1's group), - * grant rwx to "other". - * - * As user2: - * Set an xattr (should fail). - * - * As user1: - * Set an xattr (should pass). - * - * As user2: - * Read the xattr (should pass). - * Remove the xattr (should fail). - * - * As user1: - * Read the xattr (should pass). - * Remove the xattr (should pass). + * + * Test to make sure that user namespace xattrs can be set only if path has + * access and for sticky directorries, only owner/privileged user can write. + * Trusted namespace xattrs can be set only with privileged users. + * + * As user1: Create a directory (/foo) as user1, chown it to user1 (and + * user1's group), grant rwx to "other". + * + * As user2: Set an xattr (should pass with path access). + * + * As user1: Set an xattr (should pass). + * + * As user2: Read the xattr (should pass). Remove the xattr (should pass with + * path access). + * + * As user1: Read the xattr (should pass). Remove the xattr (should pass). + * + * As user1: Change permissions only to owner + * + * As User2: Set an Xattr (Should fail set with no path access) Remove an + * Xattr (Should fail with no path access) + * + * As SuperUser: Set an Xattr with Trusted (Should pass) */ @Test (timeout = 30000) public void testSetXAttrPermissionAsDifferentOwner() throws Exception { final String USER1 = "user1"; - final String GROUP1 = "mygroup1"; + final String GROUP1 = "supergroup"; final UserGroupInformation user1 = UserGroupInformation. createUserForTesting(USER1, new String[] {GROUP1}); final UserGroupInformation user2 = UserGroupInformation. createUserForTesting("user2", new String[] {"mygroup2"}); + final UserGroupInformation SUPERUSER = UserGroupInformation.getCurrentUser(); MiniDFSCluster cluster = null; PrintStream bak = null; try { @@ -2503,7 +2504,7 @@ public void testSetXAttrPermissionAsDifferentOwner() throws Exception { final ByteArrayOutputStream out = new ByteArrayOutputStream(); System.setErr(new PrintStream(out)); - // mkdir foo as user1 + //Test 1. Let user1 be owner for /foo user1.doAs(new PrivilegedExceptionAction() { @Override public Object run() throws Exception { @@ -2514,7 +2515,8 @@ public Object run() throws Exception { return null; } }); - + + //Test 2. Give access to others user1.doAs(new PrivilegedExceptionAction() { @Override public Object run() throws Exception { @@ -2527,23 +2529,21 @@ public Object run() throws Exception { } }); - // No permission to write xattr for non-owning user (user2). + // Test 3. Should be allowed to write xattr if there is a path access to + // user (user2). user2.doAs(new PrivilegedExceptionAction() { @Override public Object run() throws Exception { final int ret = ToolRunner.run(fshell, new String[]{ "-setfattr", "-n", "user.a1", "-v", "1234", "/foo"}); - assertEquals("Returned should be 1", 1, ret); - final String str = out.toString(); - assertTrue("Permission denied printed", - str.indexOf("Permission denied") != -1); + assertEquals("Returned should be 0", 0, ret); out.reset(); return null; } }); - // But there should be permission to write xattr for - // the owning user. + //Test 4. There should be permission to write xattr for + // the owning user with write permissions. user1.doAs(new PrivilegedExceptionAction() { @Override public Object run() throws Exception { @@ -2555,19 +2555,55 @@ public Object run() throws Exception { } }); - // There should be permission to read,but not to remove for - // non-owning user (user2). + // Test 5. There should be permission to read non-owning user (user2) if + // there is path access to that user and also can remove. user2.doAs(new PrivilegedExceptionAction() { @Override public Object run() throws Exception { // Read - int ret = ToolRunner.run(fshell, new String[]{ - "-getfattr", "-n", "user.a1", "/foo"}); + int ret = ToolRunner.run(fshell, new String[] { "-getfattr", "-n", + "user.a1", "/foo" }); assertEquals("Returned should be 0", 0, ret); out.reset(); // Remove - ret = ToolRunner.run(fshell, new String[]{ - "-setfattr", "-x", "user.a1", "/foo"}); + ret = ToolRunner.run(fshell, new String[] { "-setfattr", "-x", + "user.a1", "/foo" }); + assertEquals("Returned should be 0", 0, ret); + out.reset(); + return null; + } + }); + + // Test 6. There should be permission to read/remove for + // the owning user with path access. + user1.doAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + return null; + } + }); + + // Test 7. Change permission to have path access only to owner(user1) + user1.doAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + // Give access to "other" + final int ret = ToolRunner.run(fshell, new String[]{ + "-chmod", "700", "/foo"}); + assertEquals("Return should be 0", 0, ret); + out.reset(); + return null; + } + }); + + // Test 8. There should be no permissions to set for + // the non-owning user with no path access. + user2.doAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + // set + int ret = ToolRunner.run(fshell, new String[] { "-setfattr", "-n", + "user.a2", "/foo" }); assertEquals("Returned should be 1", 1, ret); final String str = out.toString(); assertTrue("Permission denied printed", @@ -2576,20 +2612,31 @@ public Object run() throws Exception { return null; } }); - - // But there should be permission to read/remove for - // the owning user. - user1.doAs(new PrivilegedExceptionAction() { + + // Test 9. There should be no permissions to remove for + // the non-owning user with no path access. + user2.doAs(new PrivilegedExceptionAction() { @Override public Object run() throws Exception { - // Read - int ret = ToolRunner.run(fshell, new String[]{ - "-getfattr", "-n", "user.a1", "/foo"}); - assertEquals("Returned should be 0", 0, ret); + // set + int ret = ToolRunner.run(fshell, new String[] { "-setfattr", "-x", + "user.a2", "/foo" }); + assertEquals("Returned should be 1", 1, ret); + final String str = out.toString(); + assertTrue("Permission denied printed", + str.indexOf("Permission denied") != -1); out.reset(); - // Remove - ret = ToolRunner.run(fshell, new String[]{ - "-setfattr", "-x", "user.a1", "/foo"}); + return null; + } + }); + + // Test 10. Superuser should be allowed to set with trusted namespace + SUPERUSER.doAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + // set + int ret = ToolRunner.run(fshell, new String[] { "-setfattr", "-n", + "trusted.a3", "/foo" }); assertEquals("Returned should be 0", 0, ret); out.reset(); return null; From 94a1462bd55ad5c71d776d22b9150003fe9ae30d Mon Sep 17 00:00:00 2001 From: Kihwal Lee Date: Sat, 28 Jun 2014 15:21:46 +0000 Subject: [PATCH 065/112] HDFS-6601. Issues in finalizing rolling upgrade when there is a layout version change. Contributed by Kihwal Lee. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1606371 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hadoop/hdfs/server/namenode/FSEditLogLoader.java | 8 +++++++- .../org/apache/hadoop/hdfs/server/namenode/FSImage.java | 7 +++++++ .../apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 1 + .../java/org/apache/hadoop/hdfs/TestRollingUpgrade.java | 4 ++++ 5 files changed, 22 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 9b5978bcd81..840123e40ab 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -730,6 +730,9 @@ Release 2.5.0 - UNRELEASED HDFS-6556. Refine XAttr permissions (umamahesh) + HDFS-6601. Issues in finalizing rolling upgrade when there is a layout + version change (kihwal) + BREAKDOWN OF HDFS-2006 SUBTASKS AND RELATED JIRAS HDFS-6299. Protobuf for XAttr and client-side implementation. (Yi Liu via umamahesh) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java index 0d4f48beaca..858cd57b23f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java @@ -739,7 +739,13 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir, } case OP_ROLLING_UPGRADE_FINALIZE: { final long finalizeTime = ((RollingUpgradeOp) op).getTime(); - fsNamesys.finalizeRollingUpgradeInternal(finalizeTime); + if (fsNamesys.isRollingUpgrade()) { + // Only do it when NN is actually doing rolling upgrade. + // We can get FINALIZE without corresponding START, if NN is restarted + // before this op is consumed and a new checkpoint is created. + fsNamesys.finalizeRollingUpgradeInternal(finalizeTime); + } + fsNamesys.getFSImage().updateStorageVersion(); fsNamesys.getFSImage().renameCheckpoint(NameNodeFile.IMAGE_ROLLBACK, NameNodeFile.IMAGE); break; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java index 09d81f66fd7..7a24a52e8cc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java @@ -1021,6 +1021,13 @@ private void waitForThreads(List threads) { } } + /** + * Update version of all storage directories. + */ + public synchronized void updateStorageVersion() throws IOException { + storage.writeAll(); + } + /** * @see #saveNamespace(FSNamesystem, Canceler) */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index a53964b8114..cc522e43177 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -7730,6 +7730,7 @@ RollingUpgradeInfo finalizeRollingUpgrade() throws IOException { // roll the edit log to make sure the standby NameNode can tail getFSImage().rollEditLog(); } + getFSImage().updateStorageVersion(); getFSImage().renameCheckpoint(NameNodeFile.IMAGE_ROLLBACK, NameNodeFile.IMAGE); } finally { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java index 32f9899ca1a..40aa37ac1a8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java @@ -390,6 +390,10 @@ public void testFinalize() throws Exception { // Once finalized, there should be no more fsimage for rollbacks. Assert.assertFalse(fsimage.hasRollbackFSImage()); + + // Should have no problem in restart and replaying edits that include + // the FINALIZE op. + dfsCluster.restartNameNode(0); } finally { if (cluster != null) { cluster.shutdown(); From b717d44b52a7ddc6586f9dd2b830422b984b5b0f Mon Sep 17 00:00:00 2001 From: Jian He Date: Sat, 28 Jun 2014 23:37:46 +0000 Subject: [PATCH 066/112] YARN-614. Changed ResourceManager to not count disk failure, node loss and RM restart towards app failures. Contributed by Xuan Gong git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1606407 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../resourcemanager/rmapp/RMAppImpl.java | 18 ++- .../rmapp/attempt/RMAppAttempt.java | 12 +- .../rmapp/attempt/RMAppAttemptImpl.java | 29 ++-- .../applicationsmanager/TestAMRestart.java | 141 ++++++++++++++++-- 5 files changed, 171 insertions(+), 32 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index ef0111f09de..c1f430a0084 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -195,6 +195,9 @@ Release 2.5.0 - UNRELEASED YARN-2171. Improved CapacityScheduling to not lock on nodemanager-count when AMs heartbeat in. (Jason Lowe via vinodkv) + YARN-614. Changed ResourceManager to not count disk failure, node loss and + RM restart towards app failures. (Xuan Gong via jianhe) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index b6ca684a739..bff41cfc219 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -687,9 +687,10 @@ private void createNewAttempt() { new RMAppAttemptImpl(appAttemptId, rmContext, scheduler, masterService, submissionContext, conf, // The newly created attempt maybe last attempt if (number of - // previously NonPreempted attempts + 1) equal to the max-attempt + // previously failed attempts(which should not include Preempted, + // hardware error and NM resync) + 1) equal to the max-attempt // limit. - maxAppAttempts == (getNumNonPreemptedAppAttempts() + 1)); + maxAppAttempts == (getNumFailedAppAttempts() + 1)); attempts.put(appAttemptId, attempt); currentAttempt = attempt; } @@ -797,7 +798,7 @@ public RMAppState transition(RMAppImpl app, RMAppEvent event) { && (app.currentAttempt.getState() == RMAppAttemptState.KILLED || app.currentAttempt.getState() == RMAppAttemptState.FINISHED || (app.currentAttempt.getState() == RMAppAttemptState.FAILED - && app.getNumNonPreemptedAppAttempts() == app.maxAppAttempts))) { + && app.getNumFailedAppAttempts() == app.maxAppAttempts))) { return RMAppState.ACCEPTED; } @@ -888,7 +889,7 @@ private String getAppAttemptFailedDiagnostics(RMAppEvent event) { msg = "Unmanaged application " + this.getApplicationId() + " failed due to " + failedEvent.getDiagnostics() + ". Failing the application."; - } else if (getNumNonPreemptedAppAttempts() >= this.maxAppAttempts) { + } else if (getNumFailedAppAttempts() >= this.maxAppAttempts) { msg = "Application " + this.getApplicationId() + " failed " + this.maxAppAttempts + " times due to " + failedEvent.getDiagnostics() + ". Failing the application."; @@ -1105,11 +1106,12 @@ public void transition(RMAppImpl app, RMAppEvent event) { }; } - private int getNumNonPreemptedAppAttempts() { + private int getNumFailedAppAttempts() { int completedAttempts = 0; - // Do not count AM preemption as attempt failure. + // Do not count AM preemption, hardware failures or NM resync + // as attempt failure. for (RMAppAttempt attempt : attempts.values()) { - if (!attempt.isPreempted()) { + if (attempt.shouldCountTowardsMaxAttemptRetry()) { completedAttempts++; } } @@ -1129,7 +1131,7 @@ public AttemptFailedTransition(RMAppState initialState) { public RMAppState transition(RMAppImpl app, RMAppEvent event) { if (!app.submissionContext.getUnmanagedAM() - && app.getNumNonPreemptedAppAttempts() < app.maxAppAttempts) { + && app.getNumFailedAppAttempts() < app.maxAppAttempts) { boolean transferStateFromPreviousAttempt = false; RMAppFailedAttemptEvent failedEvent = (RMAppFailedAttemptEvent) event; transferStateFromPreviousAttempt = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java index 42c37a93aba..68a068b9811 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java @@ -197,8 +197,14 @@ public interface RMAppAttempt extends EventHandler { ApplicationAttemptReport createApplicationAttemptReport(); /** - * Return the flag which indicates whether the attempt is preempted by the - * scheduler. + * Return the flag which indicates whether the attempt failure should be + * counted to attempt retry count. + *
      + * There failure types should not be counted to attempt retry count: + *
    • preempted by the scheduler.
      • + *
    • hardware failures, such as NM failing, lost NM and NM disk errors.
      • + *
    • killed by RM because of RM restart or failover.
      • + *
    */ - boolean isPreempted(); + boolean shouldCountTowardsMaxAttemptRetry(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 1e7693f5b71..ef572d42daa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -149,9 +149,10 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { private int amContainerExitStatus = ContainerExitStatus.INVALID; private Configuration conf; - // Since AM preemption is not counted towards AM failure count, - // even if this flag is true, a new attempt can still be re-created if this - // attempt is eventually preempted. So this flag indicates that this may be + // Since AM preemption, hardware error and NM resync are not counted towards + // AM failure count, even if this flag is true, a new attempt can still be + // re-created if this attempt is eventually failed because of preemption, + // hardware error or NM resync. So this flag indicates that this may be // last attempt. private final boolean maybeLastAttempt; private static final ExpiredTransition EXPIRED_TRANSITION = @@ -1087,12 +1088,13 @@ public void transition(RMAppAttemptImpl appAttempt, .getKeepContainersAcrossApplicationAttempts() && !appAttempt.submissionContext.getUnmanagedAM()) { // See if we should retain containers for non-unmanaged applications - if (appAttempt.isPreempted()) { - // Premption doesn't count towards app-failures and so we should - // retain containers. + if (!appAttempt.shouldCountTowardsMaxAttemptRetry()) { + // Premption, hardware failures, NM resync doesn't count towards + // app-failures and so we should retain containers. keepContainersAcrossAppAttempts = true; } else if (!appAttempt.maybeLastAttempt) { - // Not preemption. Not last-attempt too - keep containers. + // Not preemption, hardware failures or NM resync. + // Not last-attempt too - keep containers. keepContainersAcrossAppAttempts = true; } } @@ -1136,8 +1138,17 @@ public void transition(RMAppAttemptImpl appAttempt, } @Override - public boolean isPreempted() { - return getAMContainerExitStatus() == ContainerExitStatus.PREEMPTED; + public boolean shouldCountTowardsMaxAttemptRetry() { + try { + this.readLock.lock(); + int exitStatus = getAMContainerExitStatus(); + return !(exitStatus == ContainerExitStatus.PREEMPTED + || exitStatus == ContainerExitStatus.ABORTED + || exitStatus == ContainerExitStatus.DISKS_FAILED + || exitStatus == ContainerExitStatus.KILLED_BY_RESOURCEMANAGER); + } finally { + this.readLock.unlock(); + } } private static final class UnmanagedAMAttemptSavedTransition diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java index 4145b6411d8..de0987808e6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java @@ -19,13 +19,16 @@ package org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Map; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerExitStatus; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -34,6 +37,7 @@ import org.apache.hadoop.yarn.api.records.NMToken; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; @@ -49,6 +53,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.util.Records; import org.junit.Assert; import org.junit.Test; @@ -347,15 +352,20 @@ public void testNMTokensRebindOnAMRestart() throws Exception { rm1.stop(); } - // AM container preempted should not be counted towards AM max retry count. - @Test(timeout = 20000) - public void testAMPreemptedNotCountedForAMFailures() throws Exception { + // AM container preempted, nm disk failure + // should not be counted towards AM max retry count. + @Test(timeout = 100000) + public void testShouldNotCountFailureToMaxAttemptRetry() throws Exception { YarnConfiguration conf = new YarnConfiguration(); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); // explicitly set max-am-retry count as 1. conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1); - MockRM rm1 = new MockRM(conf); + conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true); + conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName()); + MemoryRMStateStore memStore = new MemoryRMStateStore(); + memStore.init(conf); + MockRM rm1 = new MockRM(conf, memStore); rm1.start(); MockNM nm1 = new MockNM("127.0.0.1:1234", 8000, rm1.getResourceTrackerService()); @@ -371,8 +381,10 @@ public void testAMPreemptedNotCountedForAMFailures() throws Exception { scheduler.killContainer(scheduler.getRMContainer(amContainer)); am1.waitForState(RMAppAttemptState.FAILED); - Assert.assertTrue(attempt1.isPreempted()); + Assert.assertTrue(! attempt1.shouldCountTowardsMaxAttemptRetry()); rm1.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED); + ApplicationState appState = + memStore.getState().getApplicationState().get(app1.getApplicationId()); // AM should be restarted even though max-am-attempt is 1. MockAM am2 = MockRM.launchAndRegisterAM(app1, rm1, nm1); RMAppAttempt attempt2 = app1.getCurrentAppAttempt(); @@ -384,20 +396,62 @@ public void testAMPreemptedNotCountedForAMFailures() throws Exception { scheduler.killContainer(scheduler.getRMContainer(amContainer2)); am2.waitForState(RMAppAttemptState.FAILED); - Assert.assertTrue(attempt2.isPreempted()); + Assert.assertTrue(! attempt2.shouldCountTowardsMaxAttemptRetry()); rm1.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED); MockAM am3 = MockRM.launchAndRegisterAM(app1, rm1, nm1); RMAppAttempt attempt3 = app1.getCurrentAppAttempt(); Assert.assertTrue(((RMAppAttemptImpl) attempt3).mayBeLastAttempt()); - // fail the AM normally - nm1.nodeHeartbeat(am3.getApplicationAttemptId(), 1, ContainerState.COMPLETE); + // mimic NM disk_failure + ContainerStatus containerStatus = Records.newRecord(ContainerStatus.class); + containerStatus.setContainerId(attempt3.getMasterContainer().getId()); + containerStatus.setDiagnostics("mimic NM disk_failure"); + containerStatus.setState(ContainerState.COMPLETE); + containerStatus.setExitStatus(ContainerExitStatus.DISKS_FAILED); + Map> conts = + new HashMap>(); + conts.put(app1.getApplicationId(), + Collections.singletonList(containerStatus)); + nm1.nodeHeartbeat(conts, true); + am3.waitForState(RMAppAttemptState.FAILED); - Assert.assertFalse(attempt3.isPreempted()); + Assert.assertTrue(! attempt3.shouldCountTowardsMaxAttemptRetry()); + Assert.assertEquals(ContainerExitStatus.DISKS_FAILED, + appState.getAttempt(am3.getApplicationAttemptId()) + .getAMContainerExitStatus()); + + rm1.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED); + MockAM am4 = MockRM.launchAndRegisterAM(app1, rm1, nm1); + RMAppAttempt attempt4 = app1.getCurrentAppAttempt(); + Assert.assertTrue(((RMAppAttemptImpl) attempt4).mayBeLastAttempt()); + + // create second NM, and register to rm1 + MockNM nm2 = + new MockNM("127.0.0.1:2234", 8000, rm1.getResourceTrackerService()); + nm2.registerNode(); + // nm1 heartbeats to report unhealthy + // This will mimic ContainerExitStatus.ABORT + nm1.nodeHeartbeat(false); + am4.waitForState(RMAppAttemptState.FAILED); + Assert.assertTrue(! attempt4.shouldCountTowardsMaxAttemptRetry()); + Assert.assertEquals(ContainerExitStatus.ABORTED, + appState.getAttempt(am4.getApplicationAttemptId()) + .getAMContainerExitStatus()); + // launch next AM in nm2 + nm2.nodeHeartbeat(true); + MockAM am5 = + rm1.waitForNewAMToLaunchAndRegister(app1.getApplicationId(), 5, nm2); + RMAppAttempt attempt5 = app1.getCurrentAppAttempt(); + Assert.assertTrue(((RMAppAttemptImpl) attempt5).mayBeLastAttempt()); + // fail the AM normally + nm2 + .nodeHeartbeat(am5.getApplicationAttemptId(), 1, ContainerState.COMPLETE); + am5.waitForState(RMAppAttemptState.FAILED); + Assert.assertTrue(attempt5.shouldCountTowardsMaxAttemptRetry()); // AM should not be restarted. rm1.waitForState(app1.getApplicationId(), RMAppState.FAILED); - Assert.assertEquals(3, app1.getAppAttempts().size()); + Assert.assertEquals(5, app1.getAppAttempts().size()); rm1.stop(); } @@ -433,7 +487,7 @@ public void testPreemptedAMRestartOnRMRestart() throws Exception { scheduler.killContainer(scheduler.getRMContainer(amContainer)); am1.waitForState(RMAppAttemptState.FAILED); - Assert.assertTrue(attempt1.isPreempted()); + Assert.assertTrue(! attempt1.shouldCountTowardsMaxAttemptRetry()); rm1.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED); // state store has 1 attempt stored. @@ -457,11 +511,74 @@ public void testPreemptedAMRestartOnRMRestart() throws Exception { RMAppAttempt attempt2 = rm2.getRMContext().getRMApps().get(app1.getApplicationId()) .getCurrentAppAttempt(); - Assert.assertFalse(attempt2.isPreempted()); + Assert.assertTrue(attempt2.shouldCountTowardsMaxAttemptRetry()); Assert.assertEquals(ContainerExitStatus.INVALID, appState.getAttempt(am2.getApplicationAttemptId()) .getAMContainerExitStatus()); rm1.stop(); rm2.stop(); } + + // Test regular RM restart/failover, new RM should not count + // AM failure towards the max-retry-account and should be able to + // re-launch the AM. + @Test(timeout = 50000) + public void testRMRestartOrFailoverNotCountedForAMFailures() + throws Exception { + YarnConfiguration conf = new YarnConfiguration(); + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true); + conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName()); + // explicitly set max-am-retry count as 1. + conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1); + MemoryRMStateStore memStore = new MemoryRMStateStore(); + memStore.init(conf); + + MockRM rm1 = new MockRM(conf, memStore); + rm1.start(); + MockNM nm1 = + new MockNM("127.0.0.1:1234", 8000, rm1.getResourceTrackerService()); + nm1.registerNode(); + RMApp app1 = rm1.submitApp(200); + // AM should be restarted even though max-am-attempt is 1. + MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); + RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); + Assert.assertTrue(((RMAppAttemptImpl) attempt1).mayBeLastAttempt()); + + // Restart rm. + MockRM rm2 = new MockRM(conf, memStore); + rm2.start(); + ApplicationState appState = + memStore.getState().getApplicationState().get(app1.getApplicationId()); + // re-register the NM + nm1.setResourceTrackerService(rm2.getResourceTrackerService()); + NMContainerStatus status = Records.newRecord(NMContainerStatus.class); + status + .setContainerExitStatus(ContainerExitStatus.KILLED_BY_RESOURCEMANAGER); + status.setContainerId(attempt1.getMasterContainer().getId()); + status.setContainerState(ContainerState.COMPLETE); + status.setDiagnostics(""); + nm1.registerNode(Collections.singletonList(status), null); + + rm2.waitForState(attempt1.getAppAttemptId(), RMAppAttemptState.FAILED); + Assert.assertEquals(ContainerExitStatus.KILLED_BY_RESOURCEMANAGER, + appState.getAttempt(am1.getApplicationAttemptId()) + .getAMContainerExitStatus()); + // Will automatically start a new AppAttempt in rm2 + rm2.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED); + MockAM am2 = + rm2.waitForNewAMToLaunchAndRegister(app1.getApplicationId(), 2, nm1); + MockRM.finishAMAndVerifyAppState(app1, rm2, nm1, am2); + RMAppAttempt attempt3 = + rm2.getRMContext().getRMApps().get(app1.getApplicationId()) + .getCurrentAppAttempt(); + Assert.assertTrue(attempt3.shouldCountTowardsMaxAttemptRetry()); + Assert.assertEquals(ContainerExitStatus.INVALID, + appState.getAttempt(am2.getApplicationAttemptId()) + .getAMContainerExitStatus()); + + rm1.stop(); + rm2.stop(); + } } From 2f75cd1190d526db8c706b0d5676324ba7a01b03 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Sun, 29 Jun 2014 16:27:24 +0000 Subject: [PATCH 067/112] HDFS-6418. Regression: DFS_NAMENODE_USER_NAME_KEY missing git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1606536 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 840123e40ab..48b6b16d83e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -733,6 +733,9 @@ Release 2.5.0 - UNRELEASED HDFS-6601. Issues in finalizing rolling upgrade when there is a layout version change (kihwal) + HDFS-6418. Regression: DFS_NAMENODE_USER_NAME_KEY missing + (szetszwo via stevel) + BREAKDOWN OF HDFS-2006 SUBTASKS AND RELATED JIRAS HDFS-6299. Protobuf for XAttr and client-side implementation. (Yi Liu via umamahesh) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index edadbd5d213..2d825effb85 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -496,16 +496,26 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_NAMENODE_STARTUP_KEY = "dfs.namenode.startup"; public static final String DFS_DATANODE_KEYTAB_FILE_KEY = "dfs.datanode.keytab.file"; public static final String DFS_DATANODE_KERBEROS_PRINCIPAL_KEY = "dfs.datanode.kerberos.principal"; + @Deprecated + public static final String DFS_DATANODE_USER_NAME_KEY = DFS_DATANODE_KERBEROS_PRINCIPAL_KEY; public static final String DFS_DATANODE_SHARED_FILE_DESCRIPTOR_PATHS = "dfs.datanode.shared.file.descriptor.paths"; public static final String DFS_DATANODE_SHARED_FILE_DESCRIPTOR_PATHS_DEFAULT = "/dev/shm,/tmp"; public static final String DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS = "dfs.short.circuit.shared.memory.watcher.interrupt.check.ms"; public static final int DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS_DEFAULT = 60000; public static final String DFS_NAMENODE_KEYTAB_FILE_KEY = "dfs.namenode.keytab.file"; public static final String DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY = "dfs.namenode.kerberos.principal"; + @Deprecated + public static final String DFS_NAMENODE_USER_NAME_KEY = DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY; public static final String DFS_NAMENODE_KERBEROS_INTERNAL_SPNEGO_PRINCIPAL_KEY = "dfs.namenode.kerberos.internal.spnego.principal"; + @Deprecated + public static final String DFS_NAMENODE_INTERNAL_SPNEGO_USER_NAME_KEY = DFS_NAMENODE_KERBEROS_INTERNAL_SPNEGO_PRINCIPAL_KEY; public static final String DFS_SECONDARY_NAMENODE_KEYTAB_FILE_KEY = "dfs.secondary.namenode.keytab.file"; public static final String DFS_SECONDARY_NAMENODE_KERBEROS_PRINCIPAL_KEY = "dfs.secondary.namenode.kerberos.principal"; + @Deprecated + public static final String DFS_SECONDARY_NAMENODE_USER_NAME_KEY = DFS_SECONDARY_NAMENODE_KERBEROS_PRINCIPAL_KEY; public static final String DFS_SECONDARY_NAMENODE_KERBEROS_INTERNAL_SPNEGO_PRINCIPAL_KEY = "dfs.secondary.namenode.kerberos.internal.spnego.principal"; + @Deprecated + public static final String DFS_SECONDARY_NAMENODE_INTERNAL_SPNEGO_USER_NAME_KEY = DFS_SECONDARY_NAMENODE_KERBEROS_INTERNAL_SPNEGO_PRINCIPAL_KEY; public static final String DFS_NAMENODE_NAME_CACHE_THRESHOLD_KEY = "dfs.namenode.name.cache.threshold"; public static final int DFS_NAMENODE_NAME_CACHE_THRESHOLD_DEFAULT = 10; public static final String DFS_NAMENODE_LEGACY_OIV_IMAGE_DIR_KEY = "dfs.namenode.legacy-oiv-image.dir"; From b0c51504c494847a5d0e98d212660297ed43ba56 Mon Sep 17 00:00:00 2001 From: Jian He Date: Sun, 29 Jun 2014 18:24:03 +0000 Subject: [PATCH 068/112] YARN-2052. Embedded an epoch number in container id to ensure the uniqueness of container id after RM restarts. Contributed by Tsuyoshi OZAWA git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1606557 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + ...erver_resourcemanager_service_protos.proto | 4 + .../server/resourcemanager/RMContext.java | 2 + .../server/resourcemanager/RMContextImpl.java | 10 +++ .../resourcemanager/ResourceManager.java | 3 + .../recovery/FileSystemRMStateStore.java | 31 +++++++- .../recovery/MemoryRMStateStore.java | 9 +++ .../recovery/NullRMStateStore.java | 5 ++ .../recovery/RMStateStore.java | 8 ++ .../recovery/ZKRMStateStore.java | 30 +++++++- .../recovery/records/Epoch.java | 74 +++++++++++++++++++ .../recovery/records/impl/pb/EpochPBImpl.java | 67 +++++++++++++++++ .../scheduler/AppSchedulingInfo.java | 16 ++-- .../SchedulerApplicationAttempt.java | 9 ++- .../TestWorkPreservingRMRestart.java | 2 +- .../recovery/RMStateStoreTestBase.java | 15 ++++ .../recovery/TestFSRMStateStore.java | 1 + .../recovery/TestZKRMStateStore.java | 1 + .../TestSchedulerApplicationAttempt.java | 8 +- .../scheduler/fair/TestFSSchedulerApp.java | 15 +++- .../fair/TestMaxRunningAppsEnforcer.java | 7 +- 21 files changed, 302 insertions(+), 18 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/Epoch.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/EpochPBImpl.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index c1f430a0084..4500097b8cc 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -45,6 +45,9 @@ Release 2.5.0 - UNRELEASED YARN-1365. Changed ApplicationMasterService to allow an app to re-register after RM restart. (Anubhav Dhoot via jianhe) + YARN-2052. Embedded an epoch number in container id to ensure the uniqueness + of container id after RM restarts. (Tsuyoshi OZAWA via jianhe) + IMPROVEMENTS YARN-1479. Invalid NaN values in Hadoop REST API JSON response (Chen He via diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto index db86d394e9f..2eb61487504 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto @@ -135,6 +135,10 @@ message RMStateVersionProto { optional int32 minor_version = 2; } +message EpochProto { + optional int64 epoch = 1; +} + ////////////////////////////////////////////////////////////////// ///////////// RM Failover related records //////////////////////// ////////////////////////////////////////////////////////////////// diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java index 517e680252f..01d506444f6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java @@ -101,4 +101,6 @@ void setRMApplicationHistoryWriter( ConfigurationProvider getConfigurationProvider(); boolean isWorkPreservingRecoveryEnabled(); + + int getEpoch(); } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java index 1abc660a824..f72ef30c012 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java @@ -82,6 +82,7 @@ public class RMContextImpl implements RMContext { private ApplicationMasterService applicationMasterService; private RMApplicationHistoryWriter rmApplicationHistoryWriter; private ConfigurationProvider configurationProvider; + private int epoch; /** * Default constructor. To be used in conjunction with setter methods for @@ -359,4 +360,13 @@ public void setConfigurationProvider( ConfigurationProvider configurationProvider) { this.configurationProvider = configurationProvider; } + + @Override + public int getEpoch() { + return this.epoch; + } + + void setEpoch(int epoch) { + this.epoch = epoch; + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 77de2090a00..c921ae9f2c9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -482,6 +482,9 @@ protected void serviceStart() throws Exception { if(recoveryEnabled) { try { rmStore.checkVersion(); + if (rmContext.isWorkPreservingRecoveryEnabled()) { + rmContext.setEpoch(rmStore.getAndIncrementEpoch()); + } RMState state = rmStore.loadState(); recover(state); } catch (Exception e) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java index 37f08cf17b1..b315a84859a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java @@ -43,15 +43,19 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.EpochProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ApplicationAttemptStateDataProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ApplicationStateDataProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RMStateVersionProto; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationAttemptStateData; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData; + +import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.Epoch; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMStateVersion; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.EpochPBImpl; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.RMStateVersionPBImpl; import org.apache.hadoop.yarn.util.ConverterUtils; @@ -71,7 +75,7 @@ public class FileSystemRMStateStore extends RMStateStore { protected static final String ROOT_DIR_NAME = "FSRMStateRoot"; protected static final RMStateVersion CURRENT_VERSION_INFO = RMStateVersion - .newInstance(1, 0); + .newInstance(1, 1); protected FileSystem fs; @@ -145,7 +149,30 @@ protected synchronized void storeVersion() throws Exception { writeFile(versionNodePath, data); } } - + + @Override + public synchronized int getAndIncrementEpoch() throws Exception { + Path epochNodePath = getNodePath(rootDirPath, EPOCH_NODE); + int currentEpoch = 0; + if (fs.exists(epochNodePath)) { + // load current epoch + FileStatus status = fs.getFileStatus(epochNodePath); + byte[] data = readFile(epochNodePath, status.getLen()); + Epoch epoch = new EpochPBImpl(EpochProto.parseFrom(data)); + currentEpoch = epoch.getEpoch(); + // increment epoch and store it + byte[] storeData = Epoch.newInstance(currentEpoch + 1).getProto() + .toByteArray(); + updateFile(epochNodePath, storeData); + } else { + // initialize epoch file with 1 for the next time. + byte[] storeData = Epoch.newInstance(currentEpoch + 1).getProto() + .toByteArray(); + writeFile(epochNodePath, storeData); + } + return currentEpoch; + } + @Override public synchronized RMState loadState() throws Exception { RMState rmState = new RMState(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java index fb0ce1a5b4a..6b5b602381c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java @@ -43,6 +43,8 @@ public class MemoryRMStateStore extends RMStateStore { RMState state = new RMState(); + private int epoch = 0; + @VisibleForTesting public RMState getState() { return state; @@ -52,6 +54,13 @@ public RMState getState() { public void checkVersion() throws Exception { } + @Override + public synchronized int getAndIncrementEpoch() throws Exception { + int currentEpoch = epoch; + epoch = epoch + 1; + return currentEpoch; + } + @Override public synchronized RMState loadState() throws Exception { // return a copy of the state to allow for modification of the real state diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java index 6a0426c0e8c..603d020f55c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java @@ -47,6 +47,11 @@ protected void closeInternal() throws Exception { // Do nothing } + @Override + public synchronized int getAndIncrementEpoch() throws Exception { + return 0; + } + @Override public RMState loadState() throws Exception { throw new UnsupportedOperationException("Cannot load state from null store"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java index b18a8748b92..9b05ea1fdcc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java @@ -45,6 +45,7 @@ import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.RMFatalEvent; @@ -85,6 +86,7 @@ public abstract class RMStateStore extends AbstractService { protected static final String DELEGATION_TOKEN_SEQUENCE_NUMBER_PREFIX = "RMDTSequenceNumber_"; protected static final String VERSION_NODE = "RMVersionNode"; + protected static final String EPOCH_NODE = "EpochNode"; public static final Log LOG = LogFactory.getLog(RMStateStore.class); @@ -520,6 +522,12 @@ public void checkVersion() throws Exception { */ protected abstract RMStateVersion getCurrentVersion(); + + /** + * Get the current epoch of RM and increment the value. + */ + public abstract int getAndIncrementEpoch() throws Exception; + /** * Blocking API * The derived class must recover state from the store and return a new diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java index 9ff128d790b..01bca39ad07 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java @@ -44,16 +44,21 @@ import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ApplicationAttemptStateDataProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ApplicationStateDataProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RMStateVersionProto; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.EpochProto; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.RMZKUtils; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationAttemptStateData; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData; + +import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.Epoch; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMStateVersion; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.EpochPBImpl; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.RMStateVersionPBImpl; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.zookeeper.CreateMode; @@ -81,7 +86,7 @@ public class ZKRMStateStore extends RMStateStore { protected static final String ROOT_ZNODE_NAME = "ZKRMStateRoot"; protected static final RMStateVersion CURRENT_VERSION_INFO = RMStateVersion - .newInstance(1, 0); + .newInstance(1, 1); private static final String RM_DELEGATION_TOKENS_ROOT_ZNODE_NAME = "RMDelegationTokensRoot"; private static final String RM_DT_SEQUENTIAL_NUMBER_ZNODE_NAME = @@ -102,6 +107,7 @@ public class ZKRMStateStore extends RMStateStore { * * ROOT_DIR_PATH * |--- VERSION_INFO + * |--- EPOCH_NODE * |--- RM_ZK_FENCING_LOCK * |--- RM_APP_ROOT * | |----- (#ApplicationId1) @@ -391,6 +397,28 @@ protected synchronized RMStateVersion loadVersion() throws Exception { return null; } + @Override + public synchronized int getAndIncrementEpoch() throws Exception { + String epochNodePath = getNodePath(zkRootNodePath, EPOCH_NODE); + int currentEpoch = 0; + if (existsWithRetries(epochNodePath, true) != null) { + // load current epoch + byte[] data = getDataWithRetries(epochNodePath, true); + Epoch epoch = new EpochPBImpl(EpochProto.parseFrom(data)); + currentEpoch = epoch.getEpoch(); + // increment epoch and store it + byte[] storeData = Epoch.newInstance(currentEpoch + 1).getProto() + .toByteArray(); + setDataWithRetries(epochNodePath, storeData, -1); + } else { + // initialize epoch node with 1 for the next time. + byte[] storeData = Epoch.newInstance(currentEpoch + 1).getProto() + .toByteArray(); + createWithRetries(epochNodePath, storeData, zkAcl, CreateMode.PERSISTENT); + } + return currentEpoch; + } + @Override public synchronized RMState loadState() throws Exception { RMState rmState = new RMState(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/Epoch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/Epoch.java new file mode 100644 index 00000000000..066878918d0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/Epoch.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.recovery.records; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.EpochProto; +import org.apache.hadoop.yarn.util.Records; + +/** + * The epoch information of RM for work-preserving restart. + * Epoch is incremented each time RM restart. It's used for assuring + * uniqueness of ContainerId. + */ +@Private +@Unstable +public abstract class Epoch { + + public static Epoch newInstance(int sequenceNumber) { + Epoch epoch = Records.newRecord(Epoch.class); + epoch.setEpoch(sequenceNumber); + return epoch; + } + + public abstract int getEpoch(); + + public abstract void setEpoch(int sequenceNumber); + + public abstract EpochProto getProto(); + + public String toString() { + return String.valueOf(getEpoch()); + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + getEpoch(); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + Epoch other = (Epoch) obj; + if (this.getEpoch() == other.getEpoch()) { + return true; + } else { + return false; + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/EpochPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/EpochPBImpl.java new file mode 100644 index 00000000000..4430672d079 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/EpochPBImpl.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb; + +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.EpochProto; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.EpochProtoOrBuilder; + + +import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.Epoch; + +public class EpochPBImpl extends Epoch { + + EpochProto proto = EpochProto.getDefaultInstance(); + EpochProto.Builder builder = null; + boolean viaProto = false; + + public EpochPBImpl() { + builder = EpochProto.newBuilder(); + } + + public EpochPBImpl(EpochProto proto) { + this.proto = proto; + viaProto = true; + } + + public EpochProto getProto() { + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = EpochProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public int getEpoch() { + EpochProtoOrBuilder p = viaProto ? proto : builder; + return (int) (p.getEpoch() & 0xffffffff); + } + + @Override + public void setEpoch(int sequentialNumber) { + maybeInitBuilder(); + builder.setEpoch(sequentialNumber); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java index d3d03fdd202..581321ca350 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java @@ -57,7 +57,10 @@ public class AppSchedulingInfo { private final String queueName; Queue queue; final String user; - private final AtomicInteger containerIdCounter = new AtomicInteger(0); + // TODO making containerIdCounter long + private final AtomicInteger containerIdCounter; + private final int EPOCH_BIT_MASK = 0x3ff; + private final int EPOCH_BIT_SHIFT = 22; final Set priorities = new TreeSet( new org.apache.hadoop.yarn.server.resourcemanager.resource.Priority.Comparator()); @@ -70,15 +73,19 @@ public class AppSchedulingInfo { /* Allocated by scheduler */ boolean pending = true; // for app metrics - + + public AppSchedulingInfo(ApplicationAttemptId appAttemptId, - String user, Queue queue, ActiveUsersManager activeUsersManager) { + String user, Queue queue, ActiveUsersManager activeUsersManager, + int epoch) { this.applicationAttemptId = appAttemptId; this.applicationId = appAttemptId.getApplicationId(); this.queue = queue; this.queueName = queue.getQueueName(); this.user = user; this.activeUsersManager = activeUsersManager; + this.containerIdCounter = new AtomicInteger( + (epoch & EPOCH_BIT_MASK) << EPOCH_BIT_SHIFT); } public ApplicationId getApplicationId() { @@ -413,9 +420,6 @@ public synchronized void transferStateFromPreviousAppSchedulingInfo( } public synchronized void recoverContainer(RMContainer rmContainer) { - // ContainerIdCounter on recovery will be addressed in YARN-2052 - this.containerIdCounter.incrementAndGet(); - QueueMetrics metrics = queue.getMetrics(); if (pending) { // If there was any container to recover, the application was diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index cf9e96258b8..3a51417cdf7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; +import com.google.common.base.Preconditions; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -40,6 +41,7 @@ import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; @@ -106,13 +108,14 @@ public class SchedulerApplicationAttempt { public SchedulerApplicationAttempt(ApplicationAttemptId applicationAttemptId, String user, Queue queue, ActiveUsersManager activeUsersManager, RMContext rmContext) { + Preconditions.checkNotNull("RMContext should not be null", rmContext); this.rmContext = rmContext; this.appSchedulingInfo = new AppSchedulingInfo(applicationAttemptId, user, queue, - activeUsersManager); + activeUsersManager, rmContext.getEpoch()); this.queue = queue; - - if (rmContext != null && rmContext.getRMApps() != null && + + if (rmContext.getRMApps() != null && rmContext.getRMApps() .containsKey(applicationAttemptId.getApplicationId())) { ApplicationSubmissionContext appSubmissionContext = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java index 89342afc976..90883ec1a75 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java @@ -218,7 +218,7 @@ public void testSchedulerRecovery() throws Exception { assertEquals(availableResources, schedulerAttempt.getHeadroom()); // *********** check appSchedulingInfo state *********** - assertEquals(4, schedulerAttempt.getNewContainerId()); + assertEquals((1 << 22) + 1, schedulerAttempt.getNewContainerId()); } private void checkCSQueue(MockRM rm, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java index 00f5f570631..49d71355908 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java @@ -495,6 +495,21 @@ public void testCheckVersion(RMStateStoreHelper stateStoreHelper) Assert.assertTrue(t instanceof RMStateVersionIncompatibleException); } } + + public void testEpoch(RMStateStoreHelper stateStoreHelper) + throws Exception { + RMStateStore store = stateStoreHelper.getRMStateStore(); + store.setRMDispatcher(new TestDispatcher()); + + int firstTimeEpoch = store.getAndIncrementEpoch(); + Assert.assertEquals(0, firstTimeEpoch); + + int secondTimeEpoch = store.getAndIncrementEpoch(); + Assert.assertEquals(1, secondTimeEpoch); + + int thirdTimeEpoch = store.getAndIncrementEpoch(); + Assert.assertEquals(2, thirdTimeEpoch); + } public void testAppDeletion(RMStateStoreHelper stateStoreHelper) throws Exception { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java index da25c5beda6..6ccaeaeaf57 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java @@ -158,6 +158,7 @@ public void testFSRMStateStore() throws Exception { .getFileSystem(conf).exists(tempAppAttemptFile)); testRMDTSecretManagerStateStore(fsTester); testCheckVersion(fsTester); + testEpoch(fsTester); testAppDeletion(fsTester); } finally { cluster.shutdown(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java index 284794b3734..d3a5475ab3c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java @@ -120,6 +120,7 @@ public void testZKRMStateStoreRealZK() throws Exception { testRMAppStateStore(zkTester); testRMDTSecretManagerStateStore(zkTester); testCheckVersion(zkTester); + testEpoch(zkTester); testAppDeletion(zkTester); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java index 93fd30027e9..20a4aa8b6ea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.*; @@ -61,10 +62,15 @@ public void testMove() { QueueMetrics newMetrics = newQueue.getMetrics(); ApplicationAttemptId appAttId = createAppAttemptId(0, 0); + RMContext rmContext = mock(RMContext.class); + when(rmContext.getEpoch()).thenReturn(3); SchedulerApplicationAttempt app = new SchedulerApplicationAttempt(appAttId, - user, oldQueue, oldQueue.getActiveUsersManager(), null); + user, oldQueue, oldQueue.getActiveUsersManager(), rmContext); oldMetrics.submitApp(user); + // confirm that containerId is calculated based on epoch. + assertEquals(app.getNewContainerId(), 0x00c00001); + // Resource request Resource requestedResource = Resource.newInstance(1536, 2); Priority requestedPriority = Priority.newInstance(2); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerApp.java index c651cb66bd8..2d5a6d4bc8c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerApp.java @@ -23,6 +23,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.util.Clock; import org.junit.Test; @@ -59,8 +60,11 @@ public void testDelayScheduling() { double rackLocalityThreshold = .6; ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1); + RMContext rmContext = Mockito.mock(RMContext.class); + Mockito.when(rmContext.getEpoch()).thenReturn(0); FSSchedulerApp schedulerApp = - new FSSchedulerApp(applicationAttemptId, "user1", queue , null, null); + new FSSchedulerApp(applicationAttemptId, "user1", queue , null, + rmContext); // Default level should be node-local assertEquals(NodeType.NODE_LOCAL, schedulerApp.getAllowedLocalityLevel( @@ -118,10 +122,12 @@ public void testDelaySchedulingForContinuousScheduling() long nodeLocalityDelayMs = 5 * 1000L; // 5 seconds long rackLocalityDelayMs = 6 * 1000L; // 6 seconds + RMContext rmContext = Mockito.mock(RMContext.class); + Mockito.when(rmContext.getEpoch()).thenReturn(0); ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1); FSSchedulerApp schedulerApp = new FSSchedulerApp(applicationAttemptId, "user1", queue, - null, null); + null, rmContext); AppSchedulable appSchedulable = Mockito.mock(AppSchedulable.class); long startTime = clock.getTime(); Mockito.when(appSchedulable.getStartTime()).thenReturn(startTime); @@ -173,9 +179,12 @@ public void testLocalityLevelWithoutDelays() { Priority prio = Mockito.mock(Priority.class); Mockito.when(prio.getPriority()).thenReturn(1); + RMContext rmContext = Mockito.mock(RMContext.class); + Mockito.when(rmContext.getEpoch()).thenReturn(0); ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1); FSSchedulerApp schedulerApp = - new FSSchedulerApp(applicationAttemptId, "user1", queue , null, null); + new FSSchedulerApp(applicationAttemptId, "user1", queue , null, + rmContext); assertEquals(NodeType.OFF_SWITCH, schedulerApp.getAllowedLocalityLevel( prio, 10, -1.0, -1.0)); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java index c1866f01cd1..cc738f5eb01 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java @@ -28,6 +28,7 @@ import java.util.Map; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.junit.Before; @@ -40,6 +41,7 @@ public class TestMaxRunningAppsEnforcer { private MaxRunningAppsEnforcer maxAppsEnforcer; private int appNum; private TestFairScheduler.MockClock clock; + private RMContext rmContext; @Before public void setup() throws Exception { @@ -59,13 +61,16 @@ public void setup() throws Exception { userMaxApps = allocConf.userMaxApps; maxAppsEnforcer = new MaxRunningAppsEnforcer(scheduler); appNum = 0; + rmContext = mock(RMContext.class); + when(rmContext.getEpoch()).thenReturn(0); } private FSSchedulerApp addApp(FSLeafQueue queue, String user) { ApplicationId appId = ApplicationId.newInstance(0l, appNum++); ApplicationAttemptId attId = ApplicationAttemptId.newInstance(appId, 0); boolean runnable = maxAppsEnforcer.canAppBeRunnable(queue, user); - FSSchedulerApp app = new FSSchedulerApp(attId, user, queue, null, null); + FSSchedulerApp app = new FSSchedulerApp(attId, user, queue, null, + rmContext); queue.addApp(app, runnable); if (runnable) { maxAppsEnforcer.trackRunnableApp(app); From e5ae7c55d184f94c72f5f43e42bc9f3dbf9a0a60 Mon Sep 17 00:00:00 2001 From: Xuan Gong Date: Mon, 30 Jun 2014 16:51:22 +0000 Subject: [PATCH 069/112] TestRMApplicationHistoryWriter sometimes fails in trunk. Contributed by Zhijie Shen git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1606835 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../resourcemanager/ahs/TestRMApplicationHistoryWriter.java | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 4500097b8cc..03efcb0b3f7 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -15,6 +15,9 @@ Trunk - Unreleased YARN-524 TestYarnVersionInfo failing if generated properties doesn't include an SVN URL. (stevel) + YARN-2216 TestRMApplicationHistoryWriter sometimes fails in trunk. + (Zhijie Shen via xgong) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java index f59cb748db4..e83a6b9cc48 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java @@ -420,7 +420,7 @@ private void testRMWritingMassiveHistory(MockRM rm) throws Exception { int waitCount = 0; int allocatedSize = allocated.size(); while (allocatedSize < request && waitCount++ < 200) { - Thread.sleep(100); + Thread.sleep(300); allocated = am.allocate(new ArrayList(), new ArrayList()).getAllocatedContainers(); From e8186a9deea3fb0a0ad69b0b527231af87bb330b Mon Sep 17 00:00:00 2001 From: Kihwal Lee Date: Mon, 30 Jun 2014 18:17:56 +0000 Subject: [PATCH 070/112] HDFS-6558. Missing newline in the description of dfsadmin -rollingUpgrade. Contributed by Chen He. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1606855 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 48b6b16d83e..4938a6efdd6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -736,6 +736,9 @@ Release 2.5.0 - UNRELEASED HDFS-6418. Regression: DFS_NAMENODE_USER_NAME_KEY missing (szetszwo via stevel) + HDFS-6558. Missing newline in the description of dfsadmin -rollingUpgrade. + (Chen He via kihwal) + BREAKDOWN OF HDFS-2006 SUBTASKS AND RELATED JIRAS HDFS-6299. Protobuf for XAttr and client-side implementation. (Yi Liu via umamahesh) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java index 2c1bf038cc4..294e91ba10b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java @@ -293,7 +293,7 @@ private static class RollingUpgradeCommand { static final String USAGE = "-"+NAME+" []"; static final String DESCRIPTION = USAGE + ":\n" + " query: query the current rolling upgrade status.\n" - + " prepare: prepare a new rolling upgrade." + + " prepare: prepare a new rolling upgrade.\n" + " finalize: finalize the current rolling upgrade."; /** Check if a command is the rollingUpgrade command From 4ac6e1d8951149e194eeab0fa4e0463d113732c4 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Mon, 30 Jun 2014 20:41:13 +0000 Subject: [PATCH 071/112] HADOOP-10710. hadoop.auth cookie is not properly constructed according to RFC2109. (Juan Yu via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1606923 13f79535-47bb-0310-9956-ffa450edef68 --- .../server/AuthenticationFilter.java | 12 +++++++--- .../server/TestAuthenticationFilter.java | 24 ++++++++----------- .../hadoop-common/CHANGES.txt | 3 +++ .../hadoop/http/TestHttpCookieFlag.java | 19 +++++++++++---- 4 files changed, 37 insertions(+), 21 deletions(-) diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java index dc7f0adeb3b..b4e0270d316 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java @@ -425,14 +425,20 @@ public Principal getUserPrincipal() { * cookie. It has no effect if its value < 0. * * XXX the following code duplicate some logic in Jetty / Servlet API, - * because of the fact that Hadoop is stuck at servlet 3.0 and jetty 6 + * because of the fact that Hadoop is stuck at servlet 2.5 and jetty 6 * right now. */ public static void createAuthCookie(HttpServletResponse resp, String token, String domain, String path, long expires, boolean isSecure) { - StringBuilder sb = new StringBuilder(AuthenticatedURL.AUTH_COOKIE).append - ("=").append(token); + StringBuilder sb = new StringBuilder(AuthenticatedURL.AUTH_COOKIE) + .append("="); + if (token != null && token.length() > 0) { + sb.append("\"") + .append(token) + .append("\""); + } + sb.append("; Version=1"); if (path != null) { sb.append("; Path=").append(path); diff --git a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java index 989025fbf87..2cc36587c85 100644 --- a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java +++ b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java @@ -531,21 +531,17 @@ public Object answer(InvocationOnMock invocation) throws Throwable { private static void parseCookieMap(String cookieHeader, HashMap cookieMap) { - for (String pair : cookieHeader.split(";")) { - String p = pair.trim(); - int idx = p.indexOf('='); - final String k, v; - if (idx == -1) { - k = p; - v = null; - } else if (idx == p.length()) { - k = p.substring(0, idx - 1); - v = null; - } else { - k = p.substring(0, idx); - v = p.substring(idx + 1); + List cookies = HttpCookie.parse(cookieHeader); + for (HttpCookie cookie : cookies) { + if (AuthenticatedURL.AUTH_COOKIE.equals(cookie.getName())) { + cookieMap.put(cookie.getName(), cookie.getValue()); + if (cookie.getPath() != null) { + cookieMap.put("Path", cookie.getPath()); + } + if (cookie.getDomain() != null) { + cookieMap.put("Domain", cookie.getDomain()); + } } - cookieMap.put(k, v); } } diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 4a3f548ab5e..d2d74a3e4a3 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -665,6 +665,9 @@ Release 2.5.0 - UNRELEASED HADOOP-10715. Remove public GraphiteSink#setWriter (Babak Behzad via raviprak) + HADOOP-10710. hadoop.auth cookie is not properly constructed according to + RFC2109. (Juan Yu via tucu) + Release 2.4.1 - 2014-06-23 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpCookieFlag.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpCookieFlag.java index d7104aaf752..75a94804824 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpCookieFlag.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpCookieFlag.java @@ -36,6 +36,8 @@ import java.net.URI; import java.net.URL; import java.security.GeneralSecurityException; +import java.net.HttpCookie; +import java.util.List; public class TestHttpCookieFlag { private static final String BASEDIR = System.getProperty("test.build.dir", @@ -116,8 +118,12 @@ public void testHttpCookie() throws IOException { .getConnectorAddress(0))); HttpURLConnection conn = (HttpURLConnection) new URL(base, "/echo").openConnection(); - Assert.assertEquals(AuthenticatedURL.AUTH_COOKIE + "=token; " + - "HttpOnly", conn.getHeaderField("Set-Cookie")); + + String header = conn.getHeaderField("Set-Cookie"); + List cookies = HttpCookie.parse(header); + Assert.assertTrue(!cookies.isEmpty()); + Assert.assertTrue(header.contains("; HttpOnly")); + Assert.assertTrue("token".equals(cookies.get(0).getValue())); } @Test @@ -127,8 +133,13 @@ public void testHttpsCookie() throws IOException, GeneralSecurityException { HttpsURLConnection conn = (HttpsURLConnection) new URL(base, "/echo").openConnection(); conn.setSSLSocketFactory(clientSslFactory.createSSLSocketFactory()); - Assert.assertEquals(AuthenticatedURL.AUTH_COOKIE + "=token; " + - "Secure; HttpOnly", conn.getHeaderField("Set-Cookie")); + + String header = conn.getHeaderField("Set-Cookie"); + List cookies = HttpCookie.parse(header); + Assert.assertTrue(!cookies.isEmpty()); + Assert.assertTrue(header.contains("; HttpOnly")); + Assert.assertTrue(cookies.get(0).getSecure()); + Assert.assertTrue("token".equals(cookies.get(0).getValue())); } @AfterClass From 0ca41a8f35e4f05bb04805a2e0a617850707b4db Mon Sep 17 00:00:00 2001 From: Chris Nauroth Date: Mon, 30 Jun 2014 20:46:44 +0000 Subject: [PATCH 072/112] HDFS-6591. while loop is executed tens of thousands of times in Hedged Read. Contributed by Liang Xie. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1606927 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../hadoop/hdfs/DFSClientFaultInjector.java | 5 + .../apache/hadoop/hdfs/DFSInputStream.java | 140 +++++++++--------- .../org/apache/hadoop/hdfs/TestPread.java | 92 +++++++++++- 4 files changed, 167 insertions(+), 73 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 4938a6efdd6..1c424753be8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -739,6 +739,9 @@ Release 2.5.0 - UNRELEASED HDFS-6558. Missing newline in the description of dfsadmin -rollingUpgrade. (Chen He via kihwal) + HDFS-6591. while loop is executed tens of thousands of times in Hedged Read + (Liang Xie via cnauroth) + BREAKDOWN OF HDFS-2006 SUBTASKS AND RELATED JIRAS HDFS-6299. Protobuf for XAttr and client-side implementation. (Yi Liu via umamahesh) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClientFaultInjector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClientFaultInjector.java index dba0c36d339..ebdc98f95f5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClientFaultInjector.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClientFaultInjector.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hdfs; +import java.util.concurrent.atomic.AtomicLong; + import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; @@ -29,6 +31,7 @@ @InterfaceAudience.Private public class DFSClientFaultInjector { public static DFSClientFaultInjector instance = new DFSClientFaultInjector(); + public static AtomicLong exceptionNum = new AtomicLong(0); public static DFSClientFaultInjector get() { return instance; @@ -47,4 +50,6 @@ public boolean failPacket() { } public void startFetchFromDatanode() {} + + public void fetchFromDatanodeException() {} } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java index 1750aa7a30b..1dcf373896a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java @@ -32,12 +32,14 @@ import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.Callable; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CompletionService; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorCompletionService; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.io.IOUtils; import org.apache.hadoop.classification.InterfaceAudience; @@ -81,6 +83,7 @@ public class DFSInputStream extends FSInputStream HasEnhancedByteBufferAccess { @VisibleForTesting public static boolean tcpReadsDisabledForTesting = false; + private long hedgedReadOpsLoopNumForTesting = 0; private final DFSClient dfsClient; private boolean closed = false; private final String src; @@ -976,20 +979,15 @@ private void fetchBlockByteRange(LocatedBlock block, long start, long end, private Callable getFromOneDataNode(final DNAddrPair datanode, final LocatedBlock block, final long start, final long end, final ByteBuffer bb, - final Map> corruptedBlockMap, - final CountDownLatch latch) { + final Map> corruptedBlockMap) { return new Callable() { @Override public ByteBuffer call() throws Exception { - try { - byte[] buf = bb.array(); - int offset = bb.position(); - actualGetFromOneDataNode(datanode, block, start, end, buf, offset, - corruptedBlockMap); - return bb; - } finally { - latch.countDown(); - } + byte[] buf = bb.array(); + int offset = bb.position(); + actualGetFromOneDataNode(datanode, block, start, end, buf, offset, + corruptedBlockMap); + return bb; } }; } @@ -1018,6 +1016,7 @@ private void actualGetFromOneDataNode(final DNAddrPair datanode, BlockReader reader = null; try { + DFSClientFaultInjector.get().fetchFromDatanodeException(); Token blockToken = block.getBlockToken(); int len = (int) (end - start + 1); reader = new BlockReaderFactory(dfsClient.getConf()). @@ -1097,35 +1096,43 @@ private void hedgedFetchBlockByteRange(LocatedBlock block, long start, Map> corruptedBlockMap) throws IOException { ArrayList> futures = new ArrayList>(); + CompletionService hedgedService = + new ExecutorCompletionService( + dfsClient.getHedgedReadsThreadPool()); ArrayList ignored = new ArrayList(); ByteBuffer bb = null; int len = (int) (end - start + 1); block = getBlockAt(block.getStartOffset(), false); - // Latch shared by all outstanding reads. First to finish closes - CountDownLatch hasReceivedResult = new CountDownLatch(1); while (true) { + // see HDFS-6591, this metric is used to verify/catch unnecessary loops + hedgedReadOpsLoopNumForTesting++; DNAddrPair chosenNode = null; - Future future = null; - // futures is null if there is no request already executing. + // there is no request already executing. if (futures.isEmpty()) { - // chooseDataNode is a commitment. If no node, we go to - // the NN to reget block locations. Only go here on first read. + // chooseDataNode is a commitment. If no node, we go to + // the NN to reget block locations. Only go here on first read. chosenNode = chooseDataNode(block, ignored); bb = ByteBuffer.wrap(buf, offset, len); - future = getHedgedReadFuture(chosenNode, block, start, end, bb, - corruptedBlockMap, hasReceivedResult); + Callable getFromDataNodeCallable = getFromOneDataNode( + chosenNode, block, start, end, bb, corruptedBlockMap); + Future firstRequest = hedgedService + .submit(getFromDataNodeCallable); + futures.add(firstRequest); try { - future.get(dfsClient.getHedgedReadTimeout(), TimeUnit.MILLISECONDS); - return; - } catch (TimeoutException e) { + Future future = hedgedService.poll( + dfsClient.getHedgedReadTimeout(), TimeUnit.MILLISECONDS); + if (future != null) { + future.get(); + return; + } if (DFSClient.LOG.isDebugEnabled()) { - DFSClient.LOG.debug("Waited " + dfsClient.getHedgedReadTimeout() + - "ms to read from " + chosenNode.info + "; spawning hedged read"); + DFSClient.LOG.debug("Waited " + dfsClient.getHedgedReadTimeout() + + "ms to read from " + chosenNode.info + + "; spawning hedged read"); } // Ignore this node on next go around. ignored.add(chosenNode.info); dfsClient.getHedgedReadMetrics().incHedgedReadOps(); - futures.add(future); continue; // no need to refresh block locations } catch (InterruptedException e) { // Ignore @@ -1133,25 +1140,31 @@ private void hedgedFetchBlockByteRange(LocatedBlock block, long start, // Ignore already logged in the call. } } else { - // We are starting up a 'hedged' read. We have a read already + // We are starting up a 'hedged' read. We have a read already // ongoing. Call getBestNodeDNAddrPair instead of chooseDataNode. // If no nodes to do hedged reads against, pass. try { - chosenNode = getBestNodeDNAddrPair(block.getLocations(), ignored); + try { + chosenNode = getBestNodeDNAddrPair(block.getLocations(), ignored); + } catch (IOException ioe) { + chosenNode = chooseDataNode(block, ignored); + } bb = ByteBuffer.allocate(len); - future = getHedgedReadFuture(chosenNode, block, start, end, bb, - corruptedBlockMap, hasReceivedResult); - futures.add(future); + Callable getFromDataNodeCallable = getFromOneDataNode( + chosenNode, block, start, end, bb, corruptedBlockMap); + Future oneMoreRequest = hedgedService + .submit(getFromDataNodeCallable); + futures.add(oneMoreRequest); } catch (IOException ioe) { if (DFSClient.LOG.isDebugEnabled()) { - DFSClient.LOG.debug("Failed getting node for hedged read: " + - ioe.getMessage()); + DFSClient.LOG.debug("Failed getting node for hedged read: " + + ioe.getMessage()); } } // if not succeeded. Submit callables for each datanode in a loop, wait // for a fixed interval and get the result from the fastest one. try { - ByteBuffer result = getFirstToComplete(futures, hasReceivedResult); + ByteBuffer result = getFirstToComplete(hedgedService, futures); // cancel the rest. cancelAll(futures); if (result.array() != buf) { // compare the array pointers @@ -1163,50 +1176,43 @@ private void hedgedFetchBlockByteRange(LocatedBlock block, long start, } return; } catch (InterruptedException ie) { - // Ignore - } catch (ExecutionException e) { - // exception already handled in the call method. getFirstToComplete - // will remove the failing future from the list. nothing more to do. + // Ignore and retry } - // We got here if exception. Ignore this node on next go around IFF + // We got here if exception. Ignore this node on next go around IFF // we found a chosenNode to hedge read against. if (chosenNode != null && chosenNode.info != null) { ignored.add(chosenNode.info); } } - // executed if we get an error from a data node - block = getBlockAt(block.getStartOffset(), false); } } - private Future getHedgedReadFuture(final DNAddrPair chosenNode, - final LocatedBlock block, long start, - final long end, final ByteBuffer bb, - final Map> corruptedBlockMap, - final CountDownLatch hasReceivedResult) { - Callable getFromDataNodeCallable = - getFromOneDataNode(chosenNode, block, start, end, bb, - corruptedBlockMap, hasReceivedResult); - return dfsClient.getHedgedReadsThreadPool().submit(getFromDataNodeCallable); + @VisibleForTesting + public long getHedgedReadOpsLoopNumForTesting() { + return hedgedReadOpsLoopNumForTesting; } - private ByteBuffer getFirstToComplete(ArrayList> futures, - CountDownLatch latch) throws ExecutionException, InterruptedException { - latch.await(); - for (Future future : futures) { - if (future.isDone()) { - try { - return future.get(); - } catch (ExecutionException e) { - // already logged in the Callable - futures.remove(future); - throw e; - } - } + private ByteBuffer getFirstToComplete( + CompletionService hedgedService, + ArrayList> futures) throws InterruptedException { + if (futures.isEmpty()) { + throw new InterruptedException("let's retry"); } - throw new InterruptedException("latch has counted down to zero but no" - + "result available yet, for safety try to request another one from" - + "outside loop, this should be rare"); + Future future = null; + try { + future = hedgedService.take(); + ByteBuffer bb = future.get(); + futures.remove(future); + return bb; + } catch (ExecutionException e) { + // already logged in the Callable + futures.remove(future); + } catch (CancellationException ce) { + // already logged in the Callable + futures.remove(future); + } + + throw new InterruptedException("let's retry"); } private void cancelAll(List> futures) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java index a1596bfb82d..4fccc99d997 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java @@ -31,12 +31,16 @@ import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.ChecksumException; import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtocol; import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset; +import org.apache.hadoop.io.IOUtils; import org.apache.log4j.Level; +import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; @@ -49,7 +53,14 @@ public class TestPread { static final long seed = 0xDEADBEEFL; static final int blockSize = 4096; - boolean simulatedStorage = false; + boolean simulatedStorage; + boolean isHedgedRead; + + @Before + public void setup() { + simulatedStorage = false; + isHedgedRead = false; + } private void writeFile(FileSystem fileSys, Path name) throws IOException { int replication = 3;// We need > 1 blocks to test out the hedged reads. @@ -73,7 +84,7 @@ private void writeFile(FileSystem fileSys, Path name) throws IOException { // now create the real file DFSTestUtil.createFile(fileSys, name, 12 * blockSize, 12 * blockSize, - blockSize, (short) 1, seed); + blockSize, (short) replication, seed); } private void checkAndEraseData(byte[] actual, int from, byte[] expected, String message) { @@ -104,8 +115,13 @@ private void doPread(FSDataInputStream stm, long position, byte[] buffer, } if (dfstm != null) { - assertEquals("Expected read statistic to be incremented", length, dfstm - .getReadStatistics().getTotalBytesRead() - totalRead); + if (isHedgedRead) { + assertTrue("Expected read statistic to be incremented", length <= dfstm + .getReadStatistics().getTotalBytesRead() - totalRead); + } else { + assertEquals("Expected read statistic to be incremented", length, dfstm + .getReadStatistics().getTotalBytesRead() - totalRead); + } } } @@ -208,7 +224,7 @@ private void datanodeRestartTest(MiniDFSCluster cluster, FileSystem fileSys, stm.readFully(0, actual); checkAndEraseData(actual, 0, expected, "Pread Datanode Restart Test"); } - + private void cleanupFile(FileSystem fileSys, Path name) throws IOException { assertTrue(fileSys.exists(name)); assertTrue(fileSys.delete(name, true)); @@ -249,6 +265,7 @@ public void testPreadDFSNoChecksum() throws IOException { */ @Test public void testHedgedPreadDFSBasic() throws IOException { + isHedgedRead = true; Configuration conf = new Configuration(); conf.setInt(DFSConfigKeys.DFS_DFSCLIENT_HEDGED_READ_THREADPOOL_SIZE, 5); conf.setLong(DFSConfigKeys.DFS_DFSCLIENT_HEDGED_READ_THRESHOLD_MILLIS, 1); @@ -257,9 +274,73 @@ public void testHedgedPreadDFSBasic() throws IOException { // transferTo. } + @Test + public void testHedgedReadLoopTooManyTimes() throws IOException { + Configuration conf = new Configuration(); + int numHedgedReadPoolThreads = 5; + final int hedgedReadTimeoutMillis = 50; + + conf.setInt(DFSConfigKeys.DFS_DFSCLIENT_HEDGED_READ_THREADPOOL_SIZE, + numHedgedReadPoolThreads); + conf.setLong(DFSConfigKeys.DFS_DFSCLIENT_HEDGED_READ_THRESHOLD_MILLIS, + hedgedReadTimeoutMillis); + // Set up the InjectionHandler + DFSClientFaultInjector.instance = Mockito + .mock(DFSClientFaultInjector.class); + DFSClientFaultInjector injector = DFSClientFaultInjector.instance; + Mockito.doAnswer(new Answer() { + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { + if (true) { + Thread.sleep(hedgedReadTimeoutMillis + 1); + if (DFSClientFaultInjector.exceptionNum.compareAndSet(0, 1)) { + System.out.println("-------------- throw Checksum Exception"); + throw new ChecksumException("ChecksumException test", 100); + } + } + return null; + } + }).when(injector).fetchFromDatanodeException(); + + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2) + .format(true).build(); + DistributedFileSystem fileSys = cluster.getFileSystem(); + DFSClient dfsClient = fileSys.getClient(); + FSDataOutputStream output = null; + DFSInputStream input = null; + String filename = "/hedgedReadMaxOut.dat"; + try { + + Path file = new Path(filename); + output = fileSys.create(file, (short) 2); + byte[] data = new byte[64 * 1024]; + output.write(data); + output.flush(); + output.write(data); + output.flush(); + output.write(data); + output.flush(); + output.close(); + byte[] buffer = new byte[64 * 1024]; + input = dfsClient.open(filename); + input.read(0, buffer, 0, 1024); + input.close(); + assertEquals(3, input.getHedgedReadOpsLoopNumForTesting()); + } catch (BlockMissingException e) { + assertTrue(false); + } finally { + IOUtils.cleanup(null, input); + IOUtils.cleanup(null, output); + fileSys.close(); + cluster.shutdown(); + Mockito.reset(injector); + } + } + @Test public void testMaxOutHedgedReadPool() throws IOException, InterruptedException, ExecutionException { + isHedgedRead = true; Configuration conf = new Configuration(); int numHedgedReadPoolThreads = 5; final int initialHedgedReadTimeoutMillis = 50000; @@ -367,7 +448,6 @@ private void dfsPreadTest(Configuration conf, boolean disableTransferTo, boolean public void testPreadDFSSimulated() throws IOException { simulatedStorage = true; testPreadDFS(); - simulatedStorage = false; } /** From 950ae82571bd05e8bac449c20bed1fa12dbfb56b Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Mon, 30 Jun 2014 20:54:56 +0000 Subject: [PATCH 073/112] HADOOP-10695. KMSClientProvider should respect a configurable timeout. (yoderme via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1606930 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 ++ .../crypto/key/kms/KMSClientProvider.java | 50 ++++++++++++++++++- .../hadoop/crypto/key/kms/server/TestKMS.java | 42 ++++++++++++++++ 3 files changed, 93 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index d2d74a3e4a3..ac117142a17 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -165,6 +165,9 @@ Trunk (Unreleased) HADOOP-10696. Add optional attributes to KeyProvider Options and Metadata. (tucu) + HADOOP-10695. KMSClientProvider should respect a configurable timeout. + (yoderme via tucu) + BUG FIXES HADOOP-9451. Fault single-layer config if node group topology is enabled. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java index 41c1f60be7f..c18e8613d08 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.crypto.key.kms; -import com.google.common.annotations.VisibleForTesting; import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; @@ -27,6 +26,7 @@ import org.apache.hadoop.security.ProviderUtils; import org.apache.hadoop.security.authentication.client.AuthenticatedURL; import org.apache.hadoop.security.authentication.client.AuthenticationException; +import org.apache.hadoop.security.authentication.client.ConnectionConfigurator; import org.apache.hadoop.security.authentication.client.PseudoAuthenticator; import org.apache.hadoop.security.ssl.SSLFactory; import org.apache.http.client.utils.URIBuilder; @@ -71,6 +71,13 @@ public class KMSClientProvider extends KeyProvider { private static final String HTTP_PUT = "PUT"; private static final String HTTP_DELETE = "DELETE"; + + private static final String CONFIG_PREFIX = "hadoop.security.kms.client."; + + /* It's possible to specify a timeout, in seconds, in the config file */ + public static final String TIMEOUT_ATTR = CONFIG_PREFIX + "timeout"; + public static final int DEFAULT_TIMEOUT = 60; + private static KeyVersion parseJSONKeyVersion(Map valueMap) { KeyVersion keyVersion = null; if (!valueMap.isEmpty()) { @@ -141,6 +148,7 @@ public static String checkNotEmpty(String s, String name) private String kmsUrl; private SSLFactory sslFactory; + private ConnectionConfigurator configurator; @Override public String toString() { @@ -149,6 +157,42 @@ public String toString() { return sb.toString(); } + /** + * This small class exists to set the timeout values for a connection + */ + private static class TimeoutConnConfigurator + implements ConnectionConfigurator { + private ConnectionConfigurator cc; + private int timeout; + + /** + * Sets the timeout and wraps another connection configurator + * @param timeout - will set both connect and read timeouts - in seconds + * @param cc - another configurator to wrap - may be null + */ + public TimeoutConnConfigurator(int timeout, ConnectionConfigurator cc) { + this.timeout = timeout; + this.cc = cc; + } + + /** + * Calls the wrapped configure() method, then sets timeouts + * @param conn the {@link HttpURLConnection} instance to configure. + * @return the connection + * @throws IOException + */ + @Override + public HttpURLConnection configure(HttpURLConnection conn) + throws IOException { + if (cc != null) { + conn = cc.configure(conn); + } + conn.setConnectTimeout(timeout * 1000); // conversion to milliseconds + conn.setReadTimeout(timeout * 1000); + return conn; + } + } + public KMSClientProvider(URI uri, Configuration conf) throws IOException { Path path = ProviderUtils.unnestUri(uri); URL url = path.toUri().toURL(); @@ -161,6 +205,8 @@ public KMSClientProvider(URI uri, Configuration conf) throws IOException { throw new IOException(ex); } } + int timeout = conf.getInt(TIMEOUT_ATTR, DEFAULT_TIMEOUT); + configurator = new TimeoutConnConfigurator(timeout, sslFactory); } private String createServiceURL(URL url) throws IOException { @@ -222,7 +268,7 @@ private HttpURLConnection createConnection(URL url, String method) HttpURLConnection conn; try { AuthenticatedURL authUrl = new AuthenticatedURL(new PseudoAuthenticator(), - sslFactory); + configurator); conn = authUrl.openConnection(url, new AuthenticatedURL.Token()); } catch (AuthenticationException ex) { throw new IOException(ex); diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java index 0959dce47e7..a3cd29db7b5 100644 --- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java +++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java @@ -38,10 +38,12 @@ import javax.security.auth.login.LoginContext; import java.io.File; import java.io.FileWriter; +import java.io.IOException; import java.io.Writer; import java.net.InetAddress; import java.net.MalformedURLException; import java.net.ServerSocket; +import java.net.SocketTimeoutException; import java.net.URI; import java.net.URL; import java.security.Principal; @@ -851,4 +853,44 @@ public Void run() throws Exception { }); } + /** + * Test the configurable timeout in the KMSClientProvider. Open up a + * socket, but don't accept connections for it. This leads to a timeout + * when the KMS client attempts to connect. + * @throws Exception + */ + @Test + public void testKMSTimeout() throws Exception { + File confDir = getTestDir(); + Configuration conf = createBaseKMSConf(confDir); + conf.setInt(KMSClientProvider.TIMEOUT_ATTR, 1); + writeConf(confDir, conf); + + ServerSocket sock; + int port; + try { + sock = new ServerSocket(0, 50, InetAddress.getByName("localhost")); + port = sock.getLocalPort(); + } catch ( Exception e ) { + /* Problem creating socket? Just bail. */ + return; + } + + URL url = new URL("http://localhost:" + port + "/kms"); + URI uri = createKMSUri(url); + + boolean caughtTimeout = false; + try { + KeyProvider kp = new KMSClientProvider(uri, conf); + kp.getKeys(); + } catch (SocketTimeoutException e) { + caughtTimeout = true; + } catch (IOException e) { + Assert.assertTrue("Caught unexpected exception" + e.toString(), false); + } + + Assert.assertTrue(caughtTimeout); + + sock.close(); + } } From e6b0f6d34658ad3442a5f7e1dd7e1acb419ee634 Mon Sep 17 00:00:00 2001 From: Chris Nauroth Date: Tue, 1 Jul 2014 16:33:04 +0000 Subject: [PATCH 074/112] HADOOP-10767. Clean up unused code in Ls shell command. Contributed by Chris Nauroth. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1607133 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++ .../src/main/java/org/apache/hadoop/fs/shell/Ls.java | 6 ------ 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index ac117142a17..0d65aa463a1 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -489,6 +489,8 @@ Release 2.5.0 - UNRELEASED HADOOP-10649. Allow overriding the default ACL for service authorization (Benoy Antony via Arpit Agarwal) + HADOOP-10767. Clean up unused code in Ls shell command. (cnauroth) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java index edc3b0a8948..6024d885afb 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java @@ -19,21 +19,16 @@ package org.apache.hadoop.fs.shell; import java.io.IOException; -import java.net.URI; import java.text.SimpleDateFormat; import java.util.Date; import java.util.LinkedList; -import java.util.Set; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import com.google.common.collect.Sets; - /** * Get a listing of all files in that match the file patterns. */ @@ -70,7 +65,6 @@ public static void registerCommands(CommandFactory factory) { protected boolean dirRecurse; protected boolean humanReadable = false; - private Set aclNotSupportedFsSet = Sets.newHashSet(); protected String formatSize(long size) { return humanReadable From 075ff276ca9e8c192717a50b0e18485afc8571a6 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Wed, 2 Jul 2014 00:23:07 +0000 Subject: [PATCH 075/112] YARN-1713. Added get-new-app and submit-app functionality to RM web services. Contributed by Varun Vasudev. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1607216 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../records/ApplicationSubmissionContext.java | 2 +- .../yarn/webapp/GenericExceptionHandler.java | 5 + .../resourcemanager/webapp/RMWebServices.java | 275 +++++ .../dao/ApplicationSubmissionContextInfo.java | 186 +++ .../dao/ContainerLaunchContextInfo.java | 117 ++ .../webapp/dao/CredentialsInfo.java | 59 + .../webapp/dao/LocalResourceInfo.java | 96 ++ .../webapp/dao/NewApplication.java | 54 + .../webapp/dao/ResourceInfo.java | 10 +- .../TestRMWebServicesAppsModification.java | 368 +++++- .../src/site/apt/ResourceManagerRest.apt.vm | 1006 ++++++++++++----- 12 files changed, 1887 insertions(+), 294 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ContainerLaunchContextInfo.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CredentialsInfo.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LocalResourceInfo.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NewApplication.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 03efcb0b3f7..f1a00cfcdc2 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -51,6 +51,9 @@ Release 2.5.0 - UNRELEASED YARN-2052. Embedded an epoch number in container id to ensure the uniqueness of container id after RM restarts. (Tsuyoshi OZAWA via jianhe) + YARN-1713. Added get-new-app and submit-app functionality to RM web services. + (Varun Vasudev via vinodkv) + IMPROVEMENTS YARN-1479. Invalid NaN values in Hadoop REST API JSON response (Chen He via diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java index 529df113c27..1ee04f04ca3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java @@ -88,7 +88,7 @@ public static ApplicationSubmissionContext newInstance( int maxAppAttempts, Resource resource, String applicationType) { return newInstance(applicationId, applicationName, queue, priority, amContainer, isUnmanagedAM, cancelTokensWhenComplete, maxAppAttempts, - resource, null, false); + resource, applicationType, false); } @Public diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java index 74180f3f3fa..1e53e022dc4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java @@ -21,10 +21,12 @@ import java.io.IOException; import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Context; import javax.ws.rs.core.Response; import javax.ws.rs.ext.ExceptionMapper; import javax.ws.rs.ext.Provider; +import javax.xml.bind.UnmarshalException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -87,6 +89,9 @@ public Response toResponse(Exception e) { s = Response.Status.BAD_REQUEST; } else if (e instanceof BadRequestException) { s = Response.Status.BAD_REQUEST; + } else if (e instanceof WebApplicationException + && e.getCause() instanceof UnmarshalException) { + s = Response.Status.BAD_REQUEST; } else { LOG.warn("INTERNAL_SERVER_ERROR", e); s = Response.Status.INTERNAL_SERVER_ERROR; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 62e7f9f8f3d..2c2a7aaed9b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.lang.reflect.UndeclaredThrowableException; import java.security.AccessControlException; +import java.nio.ByteBuffer; import java.security.PrivilegedExceptionAction; import java.util.Arrays; import java.util.Collection; @@ -36,6 +37,7 @@ import javax.servlet.http.HttpServletResponse; import javax.ws.rs.Consumes; import javax.ws.rs.GET; +import javax.ws.rs.POST; import javax.ws.rs.PUT; import javax.ws.rs.Path; import javax.ws.rs.PathParam; @@ -47,22 +49,38 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; +import org.apache.commons.codec.binary.Base64; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.DataOutputBuffer; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AuthorizationException; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; +import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueACL; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.factories.RecordFactory; @@ -81,17 +99,22 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NewApplication; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppState; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationSubmissionContextInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationStatisticsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CredentialsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FifoSchedulerInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo; @@ -758,4 +781,256 @@ private UserGroupInformation getCallerUserGroupInformation( return callerUGI; } + + /** + * Generates a new ApplicationId which is then sent to the client + * + * @param hsr + * the servlet request + * @return Response containing the app id and the maximum resource + * capabilities + * @throws AuthorizationException + * @throws IOException + * @throws InterruptedException + */ + @POST + @Path("/apps/new-application") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public Response createNewApplication(@Context HttpServletRequest hsr) + throws AuthorizationException, IOException, InterruptedException { + init(); + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr); + if (callerUGI == null) { + throw new AuthorizationException("Unable to obtain user name, " + + "user not authenticated"); + } + + NewApplication appId = createNewApplication(); + return Response.status(Status.OK).entity(appId).build(); + + } + + // reuse the code in ClientRMService to create new app + // get the new app id and submit app + // set location header with new app location + /** + * Function to submit an app to the RM + * + * @param newApp + * structure containing information to construct the + * ApplicationSubmissionContext + * @param hsr + * the servlet request + * @return Response containing the status code + * @throws AuthorizationException + * @throws IOException + * @throws InterruptedException + */ + @POST + @Path("/apps") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public Response submitApplication(ApplicationSubmissionContextInfo newApp, + @Context HttpServletRequest hsr) throws AuthorizationException, + IOException, InterruptedException { + + init(); + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr); + if (callerUGI == null) { + throw new AuthorizationException("Unable to obtain user name, " + + "user not authenticated"); + } + + ApplicationSubmissionContext appContext = + createAppSubmissionContext(newApp); + final SubmitApplicationRequest req = + SubmitApplicationRequest.newInstance(appContext); + + try { + callerUGI + .doAs(new PrivilegedExceptionAction() { + @Override + public SubmitApplicationResponse run() throws IOException, + YarnException { + return rm.getClientRMService().submitApplication(req); + } + }); + } catch (UndeclaredThrowableException ue) { + if (ue.getCause() instanceof YarnException) { + throw new BadRequestException(ue.getCause().getMessage()); + } + LOG.info("Submit app request failed", ue); + throw ue; + } + + String url = hsr.getRequestURL() + "/" + newApp.getApplicationId(); + return Response.status(Status.ACCEPTED).header(HttpHeaders.LOCATION, url) + .build(); + } + + /** + * Function that actually creates the ApplicationId by calling the + * ClientRMService + * + * @return returns structure containing the app-id and maximum resource + * capabilities + */ + private NewApplication createNewApplication() { + GetNewApplicationRequest req = + recordFactory.newRecordInstance(GetNewApplicationRequest.class); + GetNewApplicationResponse resp; + try { + resp = rm.getClientRMService().getNewApplication(req); + } catch (YarnException e) { + String msg = "Unable to create new app from RM web service"; + LOG.error(msg, e); + throw new YarnRuntimeException(msg, e); + } + NewApplication appId = + new NewApplication(resp.getApplicationId().toString(), new ResourceInfo( + resp.getMaximumResourceCapability())); + return appId; + } + + /** + * Create the actual ApplicationSubmissionContext to be submitted to the RM + * from the information provided by the user. + * + * @param newApp + * the information provided by the user + * @return returns the constructed ApplicationSubmissionContext + * @throws IOException + */ + protected ApplicationSubmissionContext createAppSubmissionContext( + ApplicationSubmissionContextInfo newApp) throws IOException { + + // create local resources and app submission context + + ApplicationId appid; + String error = + "Could not parse application id " + newApp.getApplicationId(); + try { + appid = + ConverterUtils.toApplicationId(recordFactory, + newApp.getApplicationId()); + } catch (Exception e) { + throw new BadRequestException(error); + } + ApplicationSubmissionContext appContext = + ApplicationSubmissionContext.newInstance(appid, + newApp.getApplicationName(), newApp.getQueue(), + Priority.newInstance(newApp.getPriority()), + createContainerLaunchContext(newApp), newApp.getUnmanagedAM(), + newApp.getCancelTokensWhenComplete(), newApp.getMaxAppAttempts(), + createAppSubmissionContextResource(newApp), + newApp.getApplicationType(), + newApp.getKeepContainersAcrossApplicationAttempts()); + appContext.setApplicationTags(newApp.getApplicationTags()); + + return appContext; + } + + protected Resource createAppSubmissionContextResource( + ApplicationSubmissionContextInfo newApp) throws BadRequestException { + if (newApp.getResource().getvCores() > rm.getConfig().getInt( + YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES)) { + String msg = "Requested more cores than configured max"; + throw new BadRequestException(msg); + } + if (newApp.getResource().getMemory() > rm.getConfig().getInt( + YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB)) { + String msg = "Requested more memory than configured max"; + throw new BadRequestException(msg); + } + Resource r = + Resource.newInstance(newApp.getResource().getMemory(), newApp + .getResource().getvCores()); + return r; + } + + /** + * Create the ContainerLaunchContext required for the + * ApplicationSubmissionContext. This function takes the user information and + * generates the ByteBuffer structures required by the ContainerLaunchContext + * + * @param newApp + * the information provided by the user + * @return + * @throws BadRequestException + * @throws IOException + */ + protected ContainerLaunchContext createContainerLaunchContext( + ApplicationSubmissionContextInfo newApp) throws BadRequestException, IOException { + + // create container launch context + + HashMap hmap = new HashMap(); + for (Map.Entry entry : newApp + .getContainerLaunchContextInfo().getAuxillaryServiceData().entrySet()) { + if (entry.getValue().isEmpty() == false) { + Base64 decoder = new Base64(0, null, true); + byte[] data = decoder.decode(entry.getValue()); + hmap.put(entry.getKey(), ByteBuffer.wrap(data)); + } + } + + HashMap hlr = new HashMap(); + for (Map.Entry entry : newApp + .getContainerLaunchContextInfo().getResources().entrySet()) { + LocalResourceInfo l = entry.getValue(); + LocalResource lr = + LocalResource.newInstance( + ConverterUtils.getYarnUrlFromURI(l.getUrl()), l.getType(), + l.getVisibility(), l.getSize(), l.getTimestamp()); + hlr.put(entry.getKey(), lr); + } + + DataOutputBuffer out = new DataOutputBuffer(); + Credentials cs = + createCredentials(newApp.getContainerLaunchContextInfo() + .getCredentials()); + cs.writeTokenStorageToStream(out); + ByteBuffer tokens = ByteBuffer.wrap(out.getData()); + + ContainerLaunchContext ctx = + ContainerLaunchContext.newInstance(hlr, newApp + .getContainerLaunchContextInfo().getEnvironment(), newApp + .getContainerLaunchContextInfo().getCommands(), hmap, tokens, newApp + .getContainerLaunchContextInfo().getAcls()); + + return ctx; + } + + /** + * Generate a Credentials object from the information in the CredentialsInfo + * object. + * + * @param credentials + * the CredentialsInfo provided by the user. + * @return + */ + private Credentials createCredentials(CredentialsInfo credentials) { + Credentials ret = new Credentials(); + try { + for (Map.Entry entry : credentials.getTokens().entrySet()) { + Text alias = new Text(entry.getKey()); + Token token = new Token(); + token.decodeFromUrlString(entry.getValue()); + ret.addToken(alias, token); + } + for (Map.Entry entry : credentials.getTokens().entrySet()) { + Text alias = new Text(entry.getKey()); + Base64 decoder = new Base64(0, null, true); + byte[] secret = decoder.decode(entry.getValue()); + ret.addSecretKey(alias, secret); + } + } catch (IOException ie) { + throw new BadRequestException( + "Could not parse credentials data; exception message = " + + ie.getMessage()); + } + return ret; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java new file mode 100644 index 00000000000..f7233e6e3af --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java @@ -0,0 +1,186 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.HashSet; +import java.util.Set; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlElementWrapper; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.yarn.api.records.Priority; + +/** + * Simple class to allow users to send information required to create an + * ApplicationSubmissionContext which can then be used to submit an app + * + */ +@XmlRootElement(name = "application-submission-context") +@XmlAccessorType(XmlAccessType.FIELD) +public class ApplicationSubmissionContextInfo { + + @XmlElement(name = "application-id") + String applicationId; + + @XmlElement(name = "application-name") + String applicationName; + + String queue; + int priority; + + @XmlElement(name = "am-container-spec") + ContainerLaunchContextInfo containerInfo; + + @XmlElement(name = "unmanaged-AM") + boolean isUnmanagedAM; + + @XmlElement(name = "cancel-tokens-when-complete") + boolean cancelTokensWhenComplete; + + @XmlElement(name = "max-app-attempts") + int maxAppAttempts; + + @XmlElement(name = "resource") + ResourceInfo resource; + + @XmlElement(name = "application-type") + String applicationType; + + @XmlElement(name = "keep-containers-across-application-attempts") + boolean keepContainers; + + @XmlElementWrapper(name = "application-tags") + @XmlElement(name = "tag") + Set tags; + + public ApplicationSubmissionContextInfo() { + applicationId = ""; + applicationName = ""; + containerInfo = new ContainerLaunchContextInfo(); + resource = new ResourceInfo(); + priority = Priority.UNDEFINED.getPriority(); + isUnmanagedAM = false; + cancelTokensWhenComplete = true; + keepContainers = false; + applicationType = ""; + tags = new HashSet(); + } + + public String getApplicationId() { + return applicationId; + } + + public String getApplicationName() { + return applicationName; + } + + public String getQueue() { + return queue; + } + + public int getPriority() { + return priority; + } + + public ContainerLaunchContextInfo getContainerLaunchContextInfo() { + return containerInfo; + } + + public boolean getUnmanagedAM() { + return isUnmanagedAM; + } + + public boolean getCancelTokensWhenComplete() { + return cancelTokensWhenComplete; + } + + public int getMaxAppAttempts() { + return maxAppAttempts; + } + + public ResourceInfo getResource() { + return resource; + } + + public String getApplicationType() { + return applicationType; + } + + public boolean getKeepContainersAcrossApplicationAttempts() { + return keepContainers; + } + + public Set getApplicationTags() { + return tags; + } + + public void setApplicationId(String applicationId) { + this.applicationId = applicationId; + } + + public void setApplicationName(String applicationName) { + this.applicationName = applicationName; + } + + public void setQueue(String queue) { + this.queue = queue; + } + + public void setPriority(int priority) { + this.priority = priority; + } + + public void setContainerLaunchContextInfo( + ContainerLaunchContextInfo containerLaunchContext) { + this.containerInfo = containerLaunchContext; + } + + public void setUnmanagedAM(boolean isUnmanagedAM) { + this.isUnmanagedAM = isUnmanagedAM; + } + + public void setCancelTokensWhenComplete(boolean cancelTokensWhenComplete) { + this.cancelTokensWhenComplete = cancelTokensWhenComplete; + } + + public void setMaxAppAttempts(int maxAppAttempts) { + this.maxAppAttempts = maxAppAttempts; + } + + public void setResource(ResourceInfo resource) { + this.resource = resource; + } + + public void setApplicationType(String applicationType) { + this.applicationType = applicationType; + } + + public void + setKeepContainersAcrossApplicationAttempts(boolean keepContainers) { + this.keepContainers = keepContainers; + } + + public void setApplicationTags(Set tags) { + this.tags = tags; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ContainerLaunchContextInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ContainerLaunchContextInfo.java new file mode 100644 index 00000000000..e296c3991e3 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ContainerLaunchContextInfo.java @@ -0,0 +1,117 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlElementWrapper; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.yarn.api.records.ApplicationAccessType; + +/** + * Simple class to allow users to send information required to create a + * ContainerLaunchContext which can then be used as part of the + * ApplicationSubmissionContext + * + */ +@XmlRootElement(name = "container-launch-context-info") +@XmlAccessorType(XmlAccessType.FIELD) +public class ContainerLaunchContextInfo { + + @XmlElementWrapper(name = "local-resources") + HashMap local_resources; + HashMap environment; + + @XmlElementWrapper(name = "commands") + @XmlElement(name = "command", type = String.class) + List commands; + + @XmlElementWrapper(name = "service-data") + HashMap servicedata; + + @XmlElement(name = "credentials") + CredentialsInfo credentials; + + @XmlElementWrapper(name = "application-acls") + HashMap acls; + + public ContainerLaunchContextInfo() { + local_resources = new HashMap(); + environment = new HashMap(); + commands = new ArrayList(); + servicedata = new HashMap(); + credentials = new CredentialsInfo(); + acls = new HashMap(); + } + + public Map getResources() { + return local_resources; + } + + public Map getEnvironment() { + return environment; + } + + public List getCommands() { + return commands; + } + + public Map getAuxillaryServiceData() { + return servicedata; + } + + public CredentialsInfo getCredentials() { + return credentials; + } + + public Map getAcls() { + return acls; + } + + public void setResources(HashMap resources) { + this.local_resources = resources; + } + + public void setEnvironment(HashMap environment) { + this.environment = environment; + } + + public void setCommands(List commands) { + this.commands = commands; + } + + public void setAuxillaryServiceData(HashMap serviceData) { + this.servicedata = serviceData; + } + + public void setCredentials(CredentialsInfo credentials) { + this.credentials = credentials; + } + + public void setAcls(HashMap acls) { + this.acls = acls; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CredentialsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CredentialsInfo.java new file mode 100644 index 00000000000..76c815a19a2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CredentialsInfo.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.HashMap; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElementWrapper; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement(name = "credentials-info") +@XmlAccessorType(XmlAccessType.FIELD) +public class CredentialsInfo { + + @XmlElementWrapper(name = "tokens") + HashMap tokens; + + @XmlElementWrapper(name = "secrets") + HashMap secrets; + + public CredentialsInfo() { + tokens = new HashMap(); + secrets = new HashMap(); + } + + public HashMap getTokens() { + return tokens; + } + + public HashMap getSecrets() { + return secrets; + } + + public void setTokens(HashMap tokens) { + this.tokens = tokens; + } + + public void setSecrets(HashMap secrets) { + this.secrets = secrets; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LocalResourceInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LocalResourceInfo.java new file mode 100644 index 00000000000..6e73b76fb4a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LocalResourceInfo.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.net.URI; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.yarn.api.records.LocalResourceType; +import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; + +@XmlRootElement(name = "localresources") +@XmlAccessorType(XmlAccessType.FIELD) +public class LocalResourceInfo { + + @XmlElement(name = "resource") + URI url; + LocalResourceType type; + LocalResourceVisibility visibility; + long size; + long timestamp; + String pattern; + + public URI getUrl() { + return url; + } + + public LocalResourceType getType() { + return type; + } + + public LocalResourceVisibility getVisibility() { + return visibility; + } + + public long getSize() { + return size; + } + + public long getTimestamp() { + return timestamp; + } + + public String getPattern() { + return pattern; + } + + public void setUrl(URI url) { + this.url = url; + } + + public void setType(LocalResourceType type) { + this.type = type; + } + + public void setVisibility(LocalResourceVisibility visibility) { + this.visibility = visibility; + } + + public void setSize(long size) { + if (size <= 0) { + throw new IllegalArgumentException("size must be greater than 0"); + } + this.size = size; + } + + public void setTimestamp(long timestamp) { + if (timestamp <= 0) { + throw new IllegalArgumentException("timestamp must be greater than 0"); + } + this.timestamp = timestamp; + } + + public void setPattern(String pattern) { + this.pattern = pattern; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NewApplication.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NewApplication.java new file mode 100644 index 00000000000..b5064723199 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NewApplication.java @@ -0,0 +1,54 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement(name="NewApplication") +@XmlAccessorType(XmlAccessType.FIELD) +public class NewApplication { + + @XmlElement(name="application-id") + String applicationId; + + @XmlElement(name="maximum-resource-capability") + ResourceInfo maximumResourceCapability; + + public NewApplication() { + applicationId = ""; + maximumResourceCapability = new ResourceInfo(); + } + + public NewApplication(String appId, ResourceInfo maxResources) { + applicationId = appId; + maximumResourceCapability = maxResources; + } + + public String getApplicationId() { + return applicationId; + } + + public ResourceInfo getMaximumResourceCapability() { + return maximumResourceCapability; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java index 6b4422ce61a..9510f5f5f04 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java @@ -30,7 +30,7 @@ public class ResourceInfo { int memory; int vCores; - public ResourceInfo() { + public ResourceInfo() { } public ResourceInfo(Resource res) { @@ -50,4 +50,12 @@ public int getvCores() { public String toString() { return ""; } + + public void setMemory(int memory) { + this.memory = memory; + } + + public void setvCores(int vCores) { + this.vCores = vCores; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java index 76281d47b80..4c8eeb306b8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java @@ -25,10 +25,17 @@ import java.io.IOException; import java.io.StringReader; import java.io.StringWriter; +import java.net.URI; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Enumeration; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; import java.util.Properties; +import java.util.Set; import javax.servlet.FilterConfig; import javax.servlet.ServletException; @@ -38,9 +45,15 @@ import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; +import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.authentication.server.AuthenticationFilter; import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler; +import org.apache.hadoop.yarn.api.records.ApplicationAccessType; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.LocalResourceType; +import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -54,7 +67,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppState; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationSubmissionContextInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CredentialsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; +import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.WebServicesTestUtils; import org.codehaus.jettison.json.JSONException; @@ -80,6 +97,7 @@ import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.ClientResponse.Status; import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.api.client.filter.LoggingFilter; import com.sun.jersey.api.json.JSONJAXBContext; import com.sun.jersey.api.json.JSONMarshaller; import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; @@ -461,11 +479,7 @@ public void testSingleAppKillUnauthorized() throws Exception { .constructWebResource("apps", app.getApplicationId().toString(), "state").accept(mediaType) .entity(info, MediaType.APPLICATION_XML).put(ClientResponse.class); - if (!isAuthenticationEnabled()) { - assertEquals(Status.UNAUTHORIZED, response.getClientResponseStatus()); - } else { - assertEquals(Status.FORBIDDEN, response.getClientResponseStatus()); - } + validateResponseStatus(response, Status.FORBIDDEN); } rm.stop(); return; @@ -502,4 +516,348 @@ public void tearDown() throws Exception { } super.tearDown(); } + + /** + * Helper function to wrap frequently used code. It checks the response status + * and checks if it UNAUTHORIZED if we are running with authorization turned + * off or the param passed if we are running with authorization turned on. + * + * @param response + * the ClientResponse object to be checked + * @param expectedAuthorizedMode + * the expected Status in authorized mode. + */ + public void validateResponseStatus(ClientResponse response, + Status expectedAuthorizedMode) { + validateResponseStatus(response, Status.UNAUTHORIZED, + expectedAuthorizedMode); + } + + /** + * Helper function to wrap frequently used code. It checks the response status + * and checks if it is the param expectedUnauthorizedMode if we are running + * with authorization turned off or the param expectedAuthorizedMode passed if + * we are running with authorization turned on. + * + * @param response + * the ClientResponse object to be checked + * @param expectedUnauthorizedMode + * the expected Status in unauthorized mode. + * @param expectedAuthorizedMode + * the expected Status in authorized mode. + */ + public void validateResponseStatus(ClientResponse response, + Status expectedUnauthorizedMode, Status expectedAuthorizedMode) { + if (!isAuthenticationEnabled()) { + assertEquals(expectedUnauthorizedMode, response.getClientResponseStatus()); + } else { + assertEquals(expectedAuthorizedMode, response.getClientResponseStatus()); + } + } + + // Simple test - just post to /apps/id and validate the response + @Test + public void testGetNewApplication() throws Exception { + // client().addFilter(new LoggingFilter(System.out)); + rm.start(); + String mediaTypes[] = + { MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }; + for (String acceptMedia : mediaTypes) { + testGetNewApplication(acceptMedia); + } + rm.stop(); + return; + } + + protected String testGetNewApplication(String mediaType) throws JSONException, + ParserConfigurationException, IOException, SAXException { + ClientResponse response = + this.constructWebResource("apps", "new-application").accept(mediaType) + .post(ClientResponse.class); + validateResponseStatus(response, Status.OK); + if (!isAuthenticationEnabled()) { + return ""; + } + return validateGetNewApplicationResponse(response); + } + + protected String validateGetNewApplicationResponse(ClientResponse resp) + throws JSONException, ParserConfigurationException, IOException, + SAXException { + String ret = ""; + if (resp.getType().equals(MediaType.APPLICATION_JSON_TYPE)) { + JSONObject json = resp.getEntity(JSONObject.class); + ret = validateGetNewApplicationJsonResponse(json); + } else if (resp.getType().equals(MediaType.APPLICATION_XML_TYPE)) { + String xml = resp.getEntity(String.class); + ret = validateGetNewApplicationXMLResponse(xml); + } else { + // we should not be here + assertTrue(false); + } + return ret; + } + + protected String validateGetNewApplicationJsonResponse(JSONObject json) + throws JSONException { + String appId = json.getString("application-id"); + assertTrue(appId.isEmpty() == false); + JSONObject maxResources = json.getJSONObject("maximum-resource-capability"); + long memory = maxResources.getLong("memory"); + long vCores = maxResources.getLong("vCores"); + assertTrue(memory != 0); + assertTrue(vCores != 0); + return appId; + } + + protected String validateGetNewApplicationXMLResponse(String response) + throws ParserConfigurationException, IOException, SAXException { + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(response)); + Document dom = db.parse(is); + NodeList nodes = dom.getElementsByTagName("NewApplication"); + assertEquals("incorrect number of elements", 1, nodes.getLength()); + Element element = (Element) nodes.item(0); + String appId = WebServicesTestUtils.getXmlString(element, "application-id"); + assertTrue(appId.isEmpty() == false); + NodeList maxResourceNodes = + element.getElementsByTagName("maximum-resource-capability"); + assertEquals(1, maxResourceNodes.getLength()); + Element maxResourceCapability = (Element) maxResourceNodes.item(0); + long memory = + WebServicesTestUtils.getXmlLong(maxResourceCapability, "memory"); + long vCores = + WebServicesTestUtils.getXmlLong(maxResourceCapability, "vCores"); + assertTrue(memory != 0); + assertTrue(vCores != 0); + return appId; + } + + // Test to validate the process of submitting apps - test for appropriate + // errors as well + @Test + public void testGetNewApplicationAndSubmit() throws Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); + amNodeManager.nodeHeartbeat(true); + String mediaTypes[] = + { MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }; + for (String acceptMedia : mediaTypes) { + for (String contentMedia : mediaTypes) { + testAppSubmit(acceptMedia, contentMedia); + testAppSubmitErrors(acceptMedia, contentMedia); + } + } + rm.stop(); + return; + } + + public void testAppSubmit(String acceptMedia, String contentMedia) + throws Exception { + + // create a test app and submit it via rest(after getting an app-id) then + // get the app details from the rmcontext and check that everything matches + + // client().addFilter(new LoggingFilter(System.out)); + String lrKey = "example"; + String queueName = "testqueue"; + String appName = "test"; + String appType = "test-type"; + String urlPath = "apps"; + String appId = testGetNewApplication(acceptMedia); + List commands = new ArrayList(); + commands.add("/bin/sleep 5"); + HashMap environment = new HashMap(); + environment.put("APP_VAR", "ENV_SETTING"); + HashMap acls = + new HashMap(); + acls.put(ApplicationAccessType.MODIFY_APP, "testuser1, testuser2"); + acls.put(ApplicationAccessType.VIEW_APP, "testuser3, testuser4"); + Set tags = new HashSet(); + tags.add("tag1"); + tags.add("tag 2"); + CredentialsInfo credentials = new CredentialsInfo(); + HashMap tokens = new HashMap(); + HashMap secrets = new HashMap(); + secrets.put("secret1", Base64.encodeBase64URLSafeString("secret1".getBytes("UTF8"))); + credentials.setSecrets(secrets); + credentials.setTokens(tokens); + ApplicationSubmissionContextInfo appInfo = new ApplicationSubmissionContextInfo(); + appInfo.setApplicationId(appId); + appInfo.setApplicationName(appName); + appInfo.setPriority(3); + appInfo.setMaxAppAttempts(2); + appInfo.setQueue(queueName); + appInfo.setApplicationType(appType); + HashMap lr = + new HashMap(); + LocalResourceInfo y = new LocalResourceInfo(); + y.setUrl(new URI("http://www.test.com/file.txt")); + y.setSize(100); + y.setTimestamp(System.currentTimeMillis()); + y.setType(LocalResourceType.FILE); + y.setVisibility(LocalResourceVisibility.APPLICATION); + lr.put(lrKey, y); + appInfo.getContainerLaunchContextInfo().setResources(lr); + appInfo.getContainerLaunchContextInfo().setCommands(commands); + appInfo.getContainerLaunchContextInfo().setEnvironment(environment); + appInfo.getContainerLaunchContextInfo().setAcls(acls); + appInfo.getContainerLaunchContextInfo().getAuxillaryServiceData() + .put("test", Base64.encodeBase64URLSafeString("value12".getBytes("UTF8"))); + appInfo.getContainerLaunchContextInfo().setCredentials(credentials); + appInfo.getResource().setMemory(1024); + appInfo.getResource().setvCores(1); + appInfo.setApplicationTags(tags); + + ClientResponse response = + this.constructWebResource(urlPath).accept(acceptMedia) + .entity(appInfo, contentMedia).post(ClientResponse.class); + + if (this.isAuthenticationEnabled() == false) { + assertEquals(Status.UNAUTHORIZED, response.getClientResponseStatus()); + return; + } + assertEquals(Status.ACCEPTED, response.getClientResponseStatus()); + assertTrue(response.getHeaders().getFirst(HttpHeaders.LOCATION).isEmpty() == false); + String locURL = response.getHeaders().getFirst(HttpHeaders.LOCATION); + assertTrue(locURL.indexOf("/apps/application") != -1); + appId = locURL.substring(locURL.indexOf("/apps/") + "/apps/".length()); + + WebResource res = resource().uri(new URI(locURL)); + res = res.queryParam("user.name", webserviceUserName); + response = res.get(ClientResponse.class); + assertEquals(Status.OK, response.getClientResponseStatus()); + + RMApp app = + rm.getRMContext().getRMApps() + .get(ConverterUtils.toApplicationId(appId)); + assertEquals(appName, app.getName()); + assertEquals(webserviceUserName, app.getUser()); + assertEquals(2, app.getMaxAppAttempts()); + assertEquals(queueName, app.getQueue()); + assertEquals(appType, app.getApplicationType()); + assertEquals(tags, app.getApplicationTags()); + ContainerLaunchContext ctx = + app.getApplicationSubmissionContext().getAMContainerSpec(); + assertEquals(commands, ctx.getCommands()); + assertEquals(environment, ctx.getEnvironment()); + assertEquals(acls, ctx.getApplicationACLs()); + Map appLRs = ctx.getLocalResources(); + assertTrue(appLRs.containsKey(lrKey)); + LocalResource exampleLR = appLRs.get(lrKey); + assertEquals(ConverterUtils.getYarnUrlFromURI(y.getUrl()), + exampleLR.getResource()); + assertEquals(y.getSize(), exampleLR.getSize()); + assertEquals(y.getTimestamp(), exampleLR.getTimestamp()); + assertEquals(y.getType(), exampleLR.getType()); + assertEquals(y.getPattern(), exampleLR.getPattern()); + assertEquals(y.getVisibility(), exampleLR.getVisibility()); + + response = + this.constructWebResource("apps", appId).accept(acceptMedia) + .get(ClientResponse.class); + assertEquals(Status.OK, response.getClientResponseStatus()); + return; + } + + public void testAppSubmitErrors(String acceptMedia, String contentMedia) + throws Exception { + + // submit a bunch of bad requests(correct format but bad values) via the + // REST API and make sure we get the right error response codes + + String urlPath = "apps"; + String appId = ""; + ApplicationSubmissionContextInfo appInfo = new ApplicationSubmissionContextInfo(); + ClientResponse response = + this.constructWebResource(urlPath).accept(acceptMedia) + .entity(appInfo, contentMedia).post(ClientResponse.class); + validateResponseStatus(response, Status.BAD_REQUEST); + + appId = "random"; + appInfo.setApplicationId(appId); + response = + this.constructWebResource(urlPath).accept(acceptMedia) + .entity(appInfo, contentMedia).post(ClientResponse.class); + validateResponseStatus(response, Status.BAD_REQUEST); + + appId = "random_junk"; + appInfo.setApplicationId(appId); + response = + this.constructWebResource(urlPath).accept(acceptMedia) + .entity(appInfo, contentMedia).post(ClientResponse.class); + validateResponseStatus(response, Status.BAD_REQUEST); + + // bad resource info + appInfo.getResource().setMemory( + rm.getConfig().getInt( + YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB) + 1); + appInfo.getResource().setvCores(1); + response = + this.constructWebResource(urlPath).accept(acceptMedia) + .entity(appInfo, contentMedia).post(ClientResponse.class); + + validateResponseStatus(response, Status.BAD_REQUEST); + + appInfo.getResource().setvCores( + rm.getConfig().getInt( + YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES) + 1); + appInfo.getResource().setMemory(CONTAINER_MB); + response = + this.constructWebResource(urlPath).accept(acceptMedia) + .entity(appInfo, contentMedia).post(ClientResponse.class); + validateResponseStatus(response, Status.BAD_REQUEST); + + return; + } + + @Test + public void testAppSubmitBadJsonAndXML() throws Exception { + + // submit a bunch of bad XML and JSON via the + // REST API and make sure we get error response codes + + String urlPath = "apps"; + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); + amNodeManager.nodeHeartbeat(true); + + ApplicationSubmissionContextInfo appInfo = new ApplicationSubmissionContextInfo(); + appInfo.setApplicationName("test"); + appInfo.setPriority(3); + appInfo.setMaxAppAttempts(2); + appInfo.setQueue("testqueue"); + appInfo.setApplicationType("test-type"); + HashMap lr = + new HashMap(); + LocalResourceInfo y = new LocalResourceInfo(); + y.setUrl(new URI("http://www.test.com/file.txt")); + y.setSize(100); + y.setTimestamp(System.currentTimeMillis()); + y.setType(LocalResourceType.FILE); + y.setVisibility(LocalResourceVisibility.APPLICATION); + lr.put("example", y); + appInfo.getContainerLaunchContextInfo().setResources(lr); + appInfo.getResource().setMemory(1024); + appInfo.getResource().setvCores(1); + + String body = + ""; + ClientResponse response = + this.constructWebResource(urlPath).accept(MediaType.APPLICATION_XML) + .entity(body, MediaType.APPLICATION_XML).post(ClientResponse.class); + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + body = "{\"a\" : \"b\"}"; + response = + this.constructWebResource(urlPath).accept(MediaType.APPLICATION_XML) + .entity(body, MediaType.APPLICATION_JSON).post(ClientResponse.class); + validateResponseStatus(response, Status.BAD_REQUEST); + rm.stop(); + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm index f17b1df6e1a..e419ceef577 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm @@ -1105,9 +1105,9 @@ ResourceManager REST API's. +---+ -* Cluster Applications API +* {Cluster Applications API} - With the Applications API, you can obtain a collection of resources, each of which represents an application. When you run a GET operation on this resource, you obtain a collection of Application Objects. + With the Applications API, you can obtain a collection of resources, each of which represents an application. When you run a GET operation on this resource, you obtain a collection of Application Objects. ** URI @@ -1123,7 +1123,7 @@ ResourceManager REST API's. ** Query Parameters Supported - Multiple parameters can be specified. The started and finished times have a begin and end parameter to allow you to specify ranges. For example, one could request all applications that started between 1:00am and 2:00pm on 12/19/2011 with startedTimeBegin=1324256400&startedTimeEnd=1324303200. If the Begin parameter is not specified, it defaults to 0, and if the End parameter is not specified, it defaults to infinity. + Multiple parameters can be specified for GET operations. The started and finished times have a begin and end parameter to allow you to specify ranges. For example, one could request all applications that started between 1:00am and 2:00pm on 12/19/2011 with startedTimeBegin=1324256400&startedTimeEnd=1324303200. If the Begin parameter is not specified, it defaults to 0, and if the End parameter is not specified, it defaults to infinity. ------ * state [deprecated] - state of the application @@ -1295,7 +1295,6 @@ _01_000001 0 - +---+ * Cluster Application Statistics API @@ -1427,7 +1426,7 @@ _01_000001 ** URI - Use the following URI to obtain an app object, from a application identified by the {appid} value. + Use the following URI to obtain an app object, from a application identified by the {appid} value. ------ * http:///ws/v1/cluster/apps/{appid} @@ -1538,7 +1537,7 @@ _01_000001 } +---+ - <> + <> HTTP Request: @@ -1580,287 +1579,6 @@ _01_000001 +---+ -* Cluster Application State API - - With the application state API, you can query the state of a submitted app as well kill a running app by modifying the state of a running app using a PUT request with the state set to "KILLED". To perform the PUT operation, authentication has to be setup for the RM web services. In addition, you must be authorized to kill the app. Currently you can only change the state to "KILLED"; an attempt to change the state to any other results in a 400 error response. Examples of the unauthorized and bad request errors are below. When you carry out a successful PUT, the iniital response may be a 202. You can confirm that the app is killed by repeating the PUT request until you get a 200, querying the state using the GET method or querying for app information and checking the state. In the examples below, we repeat the PUT request and get a 200 response. - - Please note that in order to kill an app, you must have an authentication filter setup for the HTTP interface. The functionality requires that a username is set in the HttpServletRequest. If no filter is setup, the response will be an "UNAUTHORIZED" response. - -** URI - ------ - * http:///ws/v1/cluster/apps/{appid}/state ------ - -** HTTP Operations Supported - ------- - * GET - * PUT ------- - -** Query Parameters Supported - ------- - None ------- - -** Elements of object - - When you make a request for the state of an app, the information returned has the following fields - -*---------------+--------------+-------------------------------+ -|| Item || Data Type || Description | -*---------------+--------------+-------------------------------+ -| state | string | The application state - can be one of "NEW", "NEW_SAVING", "SUBMITTED", "ACCEPTED", "RUNNING", "FINISHED", "FAILED", "KILLED" | -*---------------+--------------+--------------------------------+ - - -** Response Examples - - <> - - HTTP Request - ------ - GET http:///ws/v1/cluster/apps/application_1399397633663_0003/state ------ - - Response Header: - -+---+ -HTTP/1.1 200 OK -Content-Type: application/json -Transfer-Encoding: chunked -Server: Jetty(6.1.26) -+---+ - - Response Body: - -+---+ -{ - "state":"ACCEPTED" -} -+---+ - - HTTP Request - ------ - PUT http:///ws/v1/cluster/apps/application_1399397633663_0003/state ----- - - Request Body: - -+---+ -{ - "state":"KILLED" -} -+---+ - - Response Header: - -+---+ -HTTP/1.1 202 Accepted -Content-Type: application/json -Transfer-Encoding: chunked -Location: http:///ws/v1/cluster/apps/application_1399397633663_0003 -Server: Jetty(6.1.26) -+---+ - - Response Body: - -+---+ -{ - "state":"ACCEPTED" -} -+---+ - ------ - PUT http:///ws/v1/cluster/apps/application_1399397633663_0003/state ----- - - Request Body: - -+---+ -{ - "state":"KILLED" -} -+---+ - - Response Header: - -+---+ -HTTP/1.1 200 OK -Content-Type: application/json -Transfer-Encoding: chunked -Server: Jetty(6.1.26) -+---+ - - Response Body: - -+---+ -{ - "state":"KILLED" -} -+---+ - - <> - - HTTP Request - ------ - GET http:///ws/v1/cluster/apps/application_1399397633663_0003/state ------ - - Response Header: - -+---+ -HTTP/1.1 200 OK -Content-Type: application/xml -Content-Length: 99 -Server: Jetty(6.1.26) -+---+ - - Response Body: - -+---+ - - - ACCEPTED - -+---+ - - HTTP Request - ------ - PUT http:///ws/v1/cluster/apps/application_1399397633663_0003/state ----- - - Request Body: - -+---+ - - - KILLED - -+---+ - - Response Header: - -+---+ -HTTP/1.1 202 Accepted -Content-Type: application/json -Content-Length: 794 -Location: http:///ws/v1/cluster/apps/application_1399397633663_0003 -Server: Jetty(6.1.26) -+---+ - - Response Body: - -+---+ - - - ACCEPTED - -+---+ - - HTTP Request - ------ - PUT http:///ws/v1/cluster/apps/application_1399397633663_0003/state ----- - - Request Body: - -+---+ - - - KILLED - -+---+ - - Response Header: - -+---+ -HTTP/1.1 200 OK -Content-Type: application/xml -Content-Length: 917 -Server: Jetty(6.1.26) -+---+ - - Response Body: - -+---+ - - - KILLED - -+---+ - - <> - - HTTP Request - ------ - PUT http:///ws/v1/cluster/apps/application_1399397633663_0003/state ----- - - Request Body: - -+---+ - - - KILLED - -+---+ - - Response Header: - -+---+ -HTTP/1.1 403 Unauthorized -Content-Type: application/json -Transfer-Encoding: chunked -Server: Jetty(6.1.26) -+---+ - - - <> - - HTTP Request - ------ - PUT http:///ws/v1/cluster/apps/application_1399397633663_0003/state ----- - - Request Body: - -+---+ - - - RUNNING - -+---+ - - Response Header: - -+---+ -HTTP/1.1 400 -Content-Length: 295 -Content-Type: application/xml -Server: Jetty(6.1.26) -+---+ - - Response Body: - -+---+ - - - BadRequestException - java.lang.Exception: Only 'KILLED' is allowed as a target state. - org.apache.hadoop.yarn.webapp.BadRequestException - -+---+ - * Cluster Application Attempts API With the application attempts API, you can obtain a collection of resources that represent an application attempt. When you run a GET operation on this resource, you obtain a collection of App Attempt Objects. @@ -2275,3 +1993,717 @@ Server: Jetty(6.1.26) +---+ +* {Cluster Writeable APIs} + + The setions below refer to APIs which allow to create and modify applications. These APIs are currently in alpha and may change in the future. + +* {Cluster New Application API} + + With the New Application API, you can obtain an application-id which can then be used as part of the {{{Cluster_Applications_API(Submit_Application)}Cluster Submit Applications API}} to submit applications. The response also includes the maximum resource capabilities available on the cluster. + + This feature is currently in the alpha stage and may change in the future. + +** URI + +------ + * http:///ws/v1/cluster/apps/new-application +------ + +** HTTP Operations Supported + +------ + * POST +------ + +** Query Parameters Supported + +------ + * None +------ + +** Elements of the NewApplication object + + The NewApplication response contains the following elements: + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| application-id | string | The newly created application id | +*---------------+--------------+--------------------------------+ +| maximum-resource-capabilities | object | The maximum resource capabilities available on this cluster | +*---------------+--------------+--------------------------------+ + + The object contains the following elements: + + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| memory | int | The maxiumim memory available for a container | +*---------------+--------------+--------------------------------+ +| vCores | int | The maximum number of cores available for a container | +*---------------+--------------+--------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + POST http:///ws/v1/cluster/apps/new-application +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "application-id":"application_1404198295326_0001", + "maximum-resource-capability": + { + "memory":"8192", + "vCores":"32" + } +} ++---+ + + <> + + HTTP Request: + +------ + POST http:///ws/v1/cluster/apps/new-application +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 248 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + application_1404198295326_0003 + + 8192 + 32 + + ++---+ + +* {Cluster Applications API(Submit Application)} + + The Submit Applications API can be used to submit applications. In case of submitting applications, you must first obtain an application-id using the {{{Cluster_New_Application_API}Cluster New Application API}}. The application-id must be part of the request body. The response contains a URL to the application page which can be used to track the state and progress of your application. + +** URI + +------ + * http:///ws/v1/cluster/apps +------ + +** HTTP Operations Supported + +------ + * POST +------ + +** POST Response Examples + + POST requests can be used to submit apps to the ResourceManager. As mentioned above, an application-id must be obtained first. Successful submissions result in a 202 response code and a Location header specifying where to get information about the app. Please note that in order to submit an app, you must have an authentication filter setup for the HTTP interface. The functionality requires that a username is set in the HttpServletRequest. If no filter is setup, the response will be an "UNAUTHORIZED" response. + + Please note that this feature is currently in the alpha stage and may change in the future. + +*** Elements of the POST request object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| application-id | string | The application id | +*---------------+--------------+-------------------------------+ +| application-name | string | The application name | +*---------------+--------------+-------------------------------+ +| queue | string | The name of the queue to which the application should be submitted | +*---------------+--------------+-------------------------------+ +| priority | int | The priority of the application | +*---------------+--------------+-------------------------------+ +| am-container-spec | object | The application master container launch context, described below | +*---------------+--------------+-------------------------------+ +| unmanaged-AM | boolean | Is the application using an unmanaged application master | +*---------------+--------------+-------------------------------+ +| max-app-attempts | int | The max number of attempts for this application | +*---------------+--------------+-------------------------------+ +| resource | object | The resources the application master requires, described below | +*---------------+--------------+-------------------------------+ +| application-type | string | The application type(MapReduce, Pig, Hive, etc) | +*---------------+--------------+-------------------------------+ +| keep-containers-across-application-attempts | boolean | Should YARN keep the containers used by this application instead of destroying them | +*---------------+--------------+-------------------------------+ +| application-tags | object | List of application tags, please see the request examples on how to speciy the tags | +*---------------+--------------+-------------------------------+ + + Elements of the object + + The am-container-spec object should be used to provide the container launch context for the application master. + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| local-resources | object | Object describing the resources that need to be localized, described below | +*---------------+--------------+-------------------------------+ +| environment | object | Environment variables for your containers, specified as key value pairs | +*---------------+--------------+-------------------------------+ +| commands | object | The commands for launching your container, in the order in which they should be executed | +*---------------+--------------+-------------------------------+ +| service-data | object | Application specific service data; key is the name of the auxiliary servce, value is base-64 encoding of the data you wish to pass | +*---------------+--------------+-------------------------------+ +| credentials | object | The credentials required for your application to run, described below | +*---------------+--------------+-------------------------------+ +| application-acls | objec | ACLs for your application; the key can be "VIEW_APP" or "MODIFY_APP", the value is the list of users with the permissions | +*---------------+--------------+-------------------------------+ + + Elements of the object + + The object is a collection of key-value pairs. They key is an identifier for the resources to be localized and the value is the details of the resource. The elements of the value are described below: + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| resource | string | Location of the resource to be localized | +*---------------+--------------+-------------------------------+ +| type | string | Type of the resource; options are "ARCHIVE", "FILE", and "PATTERN" | +*---------------+--------------+-------------------------------+ +| visibility | string | Visibility the resource to be localized; options are "PUBLIC", "PRIVATE", and "APPLICATION" | +*---------------+--------------+-------------------------------+ +| size | long | Size of the resource to be localized | +*---------------+--------------+-------------------------------+ +| timestamp | long | Timestamp of the resource to be localized | +*---------------+--------------+-------------------------------+ + + Elements of the object + + The credentials object should be used to pass data required for the application to authenticate itself such as delegation-tokens and secrets. + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| tokens | object | Tokens that you wish to pass to your application, specified as key-value pairs. The key is an identifier for the token and the value is the token(which should be obtained using the respective web-services) | +*---------------+--------------+-------------------------------+ +| secrets | object | Secrets that you wish to use in your application, specified as key-value pairs. They key is an identifier and the value is the base-64 encoding of the secret | +*---------------+--------------+-------------------------------+ + + + Elements of the POST request body object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| memory | int | Memory required for each container | +*---------------+--------------+-------------------------------+ +| vCores | int | Virtual cores required for each container | +*---------------+--------------+-------------------------------+ + + <> + + HTTP Request: + ++---+ + POST http:///ws/v1/cluster/apps + Accept: application/json + Content-Type: application/json + { + "application-id":"application_1404203615263_0001", + "application-name":"test", + "queue":"testqueue", + "priority":"3", + "am-container-spec": + { + "local-resources": + { + "entry": + { + "key":"example", + "value": + { + "resource":"http://www.test.com/file.txt", + "type":"FILE", + "visibility":"APPLICATION", + "size":"100", + "timestamp":"1404203616003" + } + } + }, + "environment": + { + "entry": + { + "key":"APP_VAR", + "value":"ENV_SETTING" + } + }, + "commands": + { + "command":"/bin/sleep 5" + }, + "service-data": + { + "entry": + { + "key":"test", + "value":"dmFsdWUxMg" + } + }, + "credentials": + { + "tokens":null, + "secrets": + { + "entry": + { + "key":"secret1", + "value":"c2VjcmV0MQ" + } + } + }, + "application-acls": + { + "entry": + [ + { + "key":"VIEW_APP", + "value":"testuser3, testuser4" + }, + { + "key":"MODIFY_APP", + "value":"testuser1, testuser2" + } + ] + } + }, + "unmanaged-AM":"false", + "max-app-attempts":"2", + "resource": + { + "memory":"1024", + "vCores":"1" + }, + "application-type":"YARN", + "keep-containers-across-application-attempts":"false", + "application-tags": + { + "tag": + [ + "tag 2", + "tag1" + ] + } + } ++---+ + + Response Header: + ++---+ + HTTP/1.1 202 + Transfer-Encoding: chunked + Location: http:///ws/v1/cluster/apps/application_1404203615263_0001 + Content-Type: application/json + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + No response body ++---+ + + <> + + HTTP Request: + ++---+ + POST http:///ws/v1/cluster/apps + Accept: application/xml + Content-Type: application/xml + + + application_1404204891930_0002 + test + testqueue + 3 + + + + example + + http://www.test.com/file.txt + FILE + APPLICATION + 100 + 1404204892877 + + + + + + APP_VAR + ENV_SETTING + + + + /bin/sleep 5 + + + + test + dmFsdWUxMg + + + + + + + secret1 + c2VjcmV0MQ + + + + + + VIEW_APP + testuser3, testuser4 + + + MODIFY_APP + testuser1, testuser2 + + + + false + 2 + + 1024 + 1 + + YARN + false + + tag 2 + tag1 + + ++---+ + + Response Header: + ++---+ + HTTP/1.1 202 + Transfer-Encoding: chunked + Location: http:///ws/v1/cluster/apps/application_1404204891930_0002 + Content-Type: application/xml + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + No response body ++---+ + +* Cluster Application State API + + With the application state API, you can query the state of a submitted app as well kill a running app by modifying the state of a running app using a PUT request with the state set to "KILLED". To perform the PUT operation, authentication has to be setup for the RM web services. In addition, you must be authorized to kill the app. Currently you can only change the state to "KILLED"; an attempt to change the state to any other results in a 400 error response. Examples of the unauthorized and bad request errors are below. When you carry out a successful PUT, the iniital response may be a 202. You can confirm that the app is killed by repeating the PUT request until you get a 200, querying the state using the GET method or querying for app information and checking the state. In the examples below, we repeat the PUT request and get a 200 response. + + Please note that in order to kill an app, you must have an authentication filter setup for the HTTP interface. The functionality requires that a username is set in the HttpServletRequest. If no filter is setup, the response will be an "UNAUTHORIZED" response. + + This feature is currently in the alpha stage and may change in the future. + +** URI + +----- + * http:///ws/v1/cluster/apps/{appid}/state +----- + +** HTTP Operations Supported + +------ + * GET + * PUT +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of object + + When you make a request for the state of an app, the information returned has the following fields + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| state | string | The application state - can be one of "NEW", "NEW_SAVING", "SUBMITTED", "ACCEPTED", "RUNNING", "FINISHED", "FAILED", "KILLED" | +*---------------+--------------+--------------------------------+ + + +** Response Examples + + <> + + HTTP Request + +----- + GET http:///ws/v1/cluster/apps/application_1399397633663_0003/state +----- + + Response Header: + ++---+ +HTTP/1.1 200 OK +Content-Type: application/json +Transfer-Encoding: chunked +Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "state":"ACCEPTED" +} ++---+ + + HTTP Request + +----- + PUT http:///ws/v1/cluster/apps/application_1399397633663_0003/state +---- + + Request Body: + ++---+ +{ + "state":"KILLED" +} ++---+ + + Response Header: + ++---+ +HTTP/1.1 202 Accepted +Content-Type: application/json +Transfer-Encoding: chunked +Location: http:///ws/v1/cluster/apps/application_1399397633663_0003 +Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "state":"ACCEPTED" +} ++---+ + +----- + PUT http:///ws/v1/cluster/apps/application_1399397633663_0003/state +---- + + Request Body: + ++---+ +{ + "state":"KILLED" +} ++---+ + + Response Header: + ++---+ +HTTP/1.1 200 OK +Content-Type: application/json +Transfer-Encoding: chunked +Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "state":"KILLED" +} ++---+ + + <> + + HTTP Request + +----- + GET http:///ws/v1/cluster/apps/application_1399397633663_0003/state +----- + + Response Header: + ++---+ +HTTP/1.1 200 OK +Content-Type: application/xml +Content-Length: 99 +Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + ACCEPTED + ++---+ + + HTTP Request + +----- + PUT http:///ws/v1/cluster/apps/application_1399397633663_0003/state +---- + + Request Body: + ++---+ + + + KILLED + ++---+ + + Response Header: + ++---+ +HTTP/1.1 202 Accepted +Content-Type: application/json +Content-Length: 794 +Location: http:///ws/v1/cluster/apps/application_1399397633663_0003 +Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + ACCEPTED + ++---+ + + HTTP Request + +----- + PUT http:///ws/v1/cluster/apps/application_1399397633663_0003/state +---- + + Request Body: + ++---+ + + + KILLED + ++---+ + + Response Header: + ++---+ +HTTP/1.1 200 OK +Content-Type: application/xml +Content-Length: 917 +Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + KILLED + ++---+ + + <> + + HTTP Request + +----- + PUT http:///ws/v1/cluster/apps/application_1399397633663_0003/state +---- + + Request Body: + ++---+ + + + KILLED + ++---+ + + Response Header: + ++---+ +HTTP/1.1 403 Unauthorized +Content-Type: application/json +Transfer-Encoding: chunked +Server: Jetty(6.1.26) ++---+ + + + <> + + HTTP Request + +----- + PUT http:///ws/v1/cluster/apps/application_1399397633663_0003/state +---- + + Request Body: + ++---+ + + + RUNNING + ++---+ + + Response Header: + ++---+ +HTTP/1.1 400 +Content-Length: 295 +Content-Type: application/xml +Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + BadRequestException + java.lang.Exception: Only 'KILLED' is allowed as a target state. + org.apache.hadoop.yarn.webapp.BadRequestException + ++---+ + + From 03a25d2cc1a2fb124d00edf874d67e329c65a5e6 Mon Sep 17 00:00:00 2001 From: Mayank Bansal Date: Wed, 2 Jul 2014 01:54:47 +0000 Subject: [PATCH 076/112] YARN-2022 Preempting an Application Master container can be kept as least priority when multiple applications are marked for preemption by ProportionalCapacityPreemptionPolicy (Sunil G via mayank) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1607227 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 4 + .../ProportionalCapacityPreemptionPolicy.java | 89 ++++++++++++-- .../rmapp/attempt/RMAppAttemptImpl.java | 6 +- .../rmcontainer/RMContainer.java | 2 + .../rmcontainer/RMContainerImpl.java | 22 +++- .../scheduler/AbstractYarnScheduler.java | 15 +++ .../TestWorkPreservingRMRestart.java | 38 ++++++ ...tProportionalCapacityPreemptionPolicy.java | 116 +++++++++++++++++- .../attempt/TestRMAppAttemptTransitions.java | 5 + 9 files changed, 282 insertions(+), 15 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index f1a00cfcdc2..55a8d16950e 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -55,6 +55,10 @@ Release 2.5.0 - UNRELEASED (Varun Vasudev via vinodkv) IMPROVEMENTS + + YARN-2022 Preempting an Application Master container can be kept as least priority + when multiple applications are marked for preemption by + ProportionalCapacityPreemptionPolicy (Sunil G via mayank) YARN-1479. Invalid NaN values in Hadoop REST API JSON response (Chen He via jeagles) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java index 568c5abbc01..cea3d7c422f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java @@ -111,7 +111,7 @@ public class ProportionalCapacityPreemptionPolicy implements SchedulingEditPolic public static final String NATURAL_TERMINATION_FACTOR = "yarn.resourcemanager.monitor.capacity.preemption.natural_termination_factor"; - //the dispatcher to send preempt and kill events + // the dispatcher to send preempt and kill events public EventHandler dispatcher; private final Clock clock; @@ -437,8 +437,9 @@ private void resetCapacity(ResourceCalculator rc, Resource clusterResource, private Map> getContainersToPreempt( List queues, Resource clusterResource) { - Map> list = + Map> preemptMap = new HashMap>(); + List skippedAMContainerlist = new ArrayList(); for (TempQueue qT : queues) { // we act only if we are violating balance by more than @@ -449,26 +450,83 @@ private Map> getContainersToPreempt( // accounts for natural termination of containers Resource resToObtain = Resources.multiply(qT.toBePreempted, naturalTerminationFactor); + Resource skippedAMSize = Resource.newInstance(0, 0); // lock the leafqueue while we scan applications and unreserve - synchronized(qT.leafQueue) { - NavigableSet ns = - (NavigableSet) qT.leafQueue.getApplications(); + synchronized (qT.leafQueue) { + NavigableSet ns = + (NavigableSet) qT.leafQueue.getApplications(); Iterator desc = ns.descendingIterator(); qT.actuallyPreempted = Resources.clone(resToObtain); while (desc.hasNext()) { FiCaSchedulerApp fc = desc.next(); - if (Resources.lessThanOrEqual(rc, clusterResource, - resToObtain, Resources.none())) { + if (Resources.lessThanOrEqual(rc, clusterResource, resToObtain, + Resources.none())) { break; } - list.put(fc.getApplicationAttemptId(), - preemptFrom(fc, clusterResource, resToObtain)); + preemptMap.put( + fc.getApplicationAttemptId(), + preemptFrom(fc, clusterResource, resToObtain, + skippedAMContainerlist, skippedAMSize)); } + Resource maxAMCapacityForThisQueue = Resources.multiply( + Resources.multiply(clusterResource, + qT.leafQueue.getAbsoluteCapacity()), + qT.leafQueue.getMaxAMResourcePerQueuePercent()); + + // Can try preempting AMContainers (still saving atmost + // maxAMCapacityForThisQueue AMResource's) if more resources are + // required to be preempted from this Queue. + preemptAMContainers(clusterResource, preemptMap, + skippedAMContainerlist, resToObtain, skippedAMSize, + maxAMCapacityForThisQueue); } } } - return list; + return preemptMap; + } + + /** + * As more resources are needed for preemption, saved AMContainers has to be + * rescanned. Such AMContainers can be preempted based on resToObtain, but + * maxAMCapacityForThisQueue resources will be still retained. + * + * @param clusterResource + * @param preemptMap + * @param skippedAMContainerlist + * @param resToObtain + * @param skippedAMSize + * @param maxAMCapacityForThisQueue + */ + private void preemptAMContainers(Resource clusterResource, + Map> preemptMap, + List skippedAMContainerlist, Resource resToObtain, + Resource skippedAMSize, Resource maxAMCapacityForThisQueue) { + for (RMContainer c : skippedAMContainerlist) { + // Got required amount of resources for preemption, can stop now + if (Resources.lessThanOrEqual(rc, clusterResource, resToObtain, + Resources.none())) { + break; + } + // Once skippedAMSize reaches down to maxAMCapacityForThisQueue, + // container selection iteration for preemption will be stopped. + if (Resources.lessThanOrEqual(rc, clusterResource, skippedAMSize, + maxAMCapacityForThisQueue)) { + break; + } + Set contToPrempt = preemptMap.get(c + .getApplicationAttemptId()); + if (null == contToPrempt) { + contToPrempt = new HashSet(); + preemptMap.put(c.getApplicationAttemptId(), contToPrempt); + } + contToPrempt.add(c); + + Resources.subtractFrom(resToObtain, c.getContainer().getResource()); + Resources.subtractFrom(skippedAMSize, c.getContainer() + .getResource()); + } + skippedAMContainerlist.clear(); } /** @@ -480,8 +538,9 @@ private Map> getContainersToPreempt( * @param rsrcPreempt * @return */ - private Set preemptFrom( - FiCaSchedulerApp app, Resource clusterResource, Resource rsrcPreempt) { + private Set preemptFrom(FiCaSchedulerApp app, + Resource clusterResource, Resource rsrcPreempt, + List skippedAMContainerlist, Resource skippedAMSize) { Set ret = new HashSet(); ApplicationAttemptId appId = app.getApplicationAttemptId(); @@ -513,6 +572,12 @@ private Set preemptFrom( rsrcPreempt, Resources.none())) { return ret; } + // Skip AM Container from preemption for now. + if (c.isAMContainer()) { + skippedAMContainerlist.add(c); + Resources.addTo(skippedAMSize, c.getContainer().getResource()); + continue; + } ret.add(c); Resources.subtractFrom(rsrcPreempt, c.getContainer().getResource()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index ef572d42daa..626d47efb3e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -84,6 +84,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptStatusupdateEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUnregistrationEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUpdateSavedEvent; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; @@ -832,7 +833,10 @@ public RMAppAttemptState transition(RMAppAttemptImpl appAttempt, // Set the masterContainer appAttempt.setMasterContainer(amContainerAllocation.getContainers() - .get(0)); + .get(0)); + RMContainerImpl rmMasterContainer = (RMContainerImpl)appAttempt.scheduler + .getRMContainer(appAttempt.getMasterContainer().getId()); + rmMasterContainer.setAMContainer(true); // The node set in NMTokenSecrentManager is used for marking whether the // NMToken has been issued for this node to the AM. // When AM container was allocated to RM itself, the node which allocates diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java index 51bd80c7ad0..96150c3c2ae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java @@ -71,5 +71,7 @@ public interface RMContainer extends EventHandler { ContainerState getContainerState(); ContainerReport createContainerReport(); + + boolean isAMContainer(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java index 6eb8c3dc5b1..b8d2f5536a2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java @@ -155,6 +155,7 @@ RMContainerEventType.RELEASED, new KillTransition()) private long creationTime; private long finishTime; private ContainerStatus finishedStatus; + private boolean isAMContainer; public RMContainerImpl(Container container, ApplicationAttemptId appAttemptId, NodeId nodeId, String user, @@ -176,6 +177,7 @@ public RMContainerImpl(Container container, this.rmContext = rmContext; this.eventHandler = rmContext.getDispatcher().getEventHandler(); this.containerAllocationExpirer = rmContext.getContainerAllocationExpirer(); + this.isAMContainer = false; ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); this.readLock = lock.readLock(); @@ -313,6 +315,25 @@ public String toString() { return containerId.toString(); } + @Override + public boolean isAMContainer() { + try { + readLock.lock(); + return isAMContainer; + } finally { + readLock.unlock(); + } + } + + public void setAMContainer(boolean isAMContainer) { + try { + writeLock.lock(); + this.isAMContainer = isAMContainer; + } finally { + writeLock.unlock(); + } + } + @Override public void handle(RMContainerEvent event) { LOG.debug("Processing " + event.getContainerId() + " of type " + event.getType()); @@ -490,5 +511,4 @@ public ContainerReport createContainerReport() { } return containerReport; } - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index 15ba23cc95b..38678845ec1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -39,6 +39,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerRecoverEvent; @@ -242,6 +243,20 @@ public synchronized void recoverContainersOnNode( // recover scheduler attempt schedulerAttempt.recoverContainer(rmContainer); + + // set master container for the current running AMContainer for this + // attempt. + RMAppAttempt appAttempt = rmApp.getCurrentAppAttempt(); + if (appAttempt != null) { + Container masterContainer = appAttempt.getMasterContainer(); + + // Mark current running AMContainer's RMContainer based on the master + // container ID stored in AppAttempt. + if (masterContainer != null + && masterContainer.getId().equals(rmContainer.getContainerId())) { + ((RMContainerImpl)rmContainer).setAMContainer(true); + } + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java index 90883ec1a75..fb5c3a35ae3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java @@ -62,6 +62,7 @@ import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -564,6 +565,43 @@ public void testAppReregisterOnRMWorkPreservingRestart() throws Exception { rm2.waitForState(app0.getApplicationId(), RMAppState.RUNNING); rm2.waitForState(am0.getApplicationAttemptId(), RMAppAttemptState.RUNNING); } + + @Test (timeout = 30000) + public void testAMContainerStatusWithRMRestart() throws Exception { + MemoryRMStateStore memStore = new MemoryRMStateStore(); + memStore.init(conf); + rm1 = new MockRM(conf, memStore); + rm1.start(); + MockNM nm1 = + new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService()); + nm1.registerNode(); + RMApp app1_1 = rm1.submitApp(1024); + MockAM am1_1 = MockRM.launchAndRegisterAM(app1_1, rm1, nm1); + + RMAppAttempt attempt0 = app1_1.getCurrentAppAttempt(); + AbstractYarnScheduler scheduler = + ((AbstractYarnScheduler) rm1.getResourceScheduler()); + + Assert.assertTrue(scheduler.getRMContainer( + attempt0.getMasterContainer().getId()).isAMContainer()); + + // Re-start RM + rm2 = new MockRM(conf, memStore); + rm2.start(); + nm1.setResourceTrackerService(rm2.getResourceTrackerService()); + + List am1_1Containers = + createNMContainerStatusForApp(am1_1); + nm1.registerNode(am1_1Containers, null); + + // Wait for RM to settle down on recovering containers; + waitForNumContainersToRecover(2, rm2, am1_1.getApplicationAttemptId()); + + scheduler = ((AbstractYarnScheduler) rm2.getResourceScheduler()); + Assert.assertTrue(scheduler.getRMContainer( + attempt0.getMasterContainer().getId()).isAMContainer()); + } + private void asserteMetrics(QueueMetrics qm, int appsSubmitted, int appsPending, int appsRunning, int appsCompleted, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java index d0a80eb20bb..8a2840e8632 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java @@ -80,6 +80,8 @@ public class TestProportionalCapacityPreemptionPolicy { static final long TS = 3141592653L; int appAlloc = 0; + boolean setAMContainer = false; + float setAMResourcePercent = 0.0f; Random rand = null; Clock mClock = null; Configuration conf = null; @@ -466,7 +468,108 @@ public void testPolicyInitializeAfterSchedulerInitialized() { fail("Failed to find SchedulingMonitor service, please check what happened"); } + + @Test + public void testSkipAMContainer() { + int[][] qData = new int[][] { + // / A B + { 100, 50, 50 }, // abs + { 100, 100, 100 }, // maxcap + { 100, 100, 0 }, // used + { 70, 20, 50 }, // pending + { 0, 0, 0 }, // reserved + { 5, 4, 1 }, // apps + { -1, 1, 1 }, // req granularity + { 2, 0, 0 }, // subqueues + }; + setAMContainer = true; + ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData); + policy.editSchedule(); + + // By skipping AM Container, all other 24 containers of appD will be + // preempted + verify(mDisp, times(24)).handle(argThat(new IsPreemptionRequestFor(appD))); + // By skipping AM Container, all other 24 containers of appC will be + // preempted + verify(mDisp, times(24)).handle(argThat(new IsPreemptionRequestFor(appC))); + + // Since AM containers of appC and appD are saved, 2 containers from appB + // has to be preempted. + verify(mDisp, times(2)).handle(argThat(new IsPreemptionRequestFor(appB))); + setAMContainer = false; + } + + @Test + public void testPreemptSkippedAMContainers() { + int[][] qData = new int[][] { + // / A B + { 100, 10, 90 }, // abs + { 100, 100, 100 }, // maxcap + { 100, 100, 0 }, // used + { 70, 20, 90 }, // pending + { 0, 0, 0 }, // reserved + { 5, 4, 1 }, // apps + { -1, 5, 5 }, // req granularity + { 2, 0, 0 }, // subqueues + }; + setAMContainer = true; + ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData); + policy.editSchedule(); + + // All 5 containers of appD will be preempted including AM container. + verify(mDisp, times(5)).handle(argThat(new IsPreemptionRequestFor(appD))); + + // All 5 containers of appC will be preempted including AM container. + verify(mDisp, times(5)).handle(argThat(new IsPreemptionRequestFor(appC))); + + // By skipping AM Container, all other 4 containers of appB will be + // preempted + verify(mDisp, times(4)).handle(argThat(new IsPreemptionRequestFor(appB))); + + // By skipping AM Container, all other 4 containers of appA will be + // preempted + verify(mDisp, times(4)).handle(argThat(new IsPreemptionRequestFor(appA))); + setAMContainer = false; + } + + @Test + public void testAMResourcePercentForSkippedAMContainers() { + int[][] qData = new int[][] { + // / A B + { 100, 10, 90 }, // abs + { 100, 100, 100 }, // maxcap + { 100, 100, 0 }, // used + { 70, 20, 90 }, // pending + { 0, 0, 0 }, // reserved + { 5, 4, 1 }, // apps + { -1, 5, 5 }, // req granularity + { 2, 0, 0 }, // subqueues + }; + setAMContainer = true; + setAMResourcePercent = 0.5f; + ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData); + policy.editSchedule(); + + // AMResoucePercent is 50% of cluster and maxAMCapacity will be 5Gb. + // Total used AM container size is 20GB, hence 2 AM container has + // to be preempted as Queue Capacity is 10Gb. + verify(mDisp, times(5)).handle(argThat(new IsPreemptionRequestFor(appD))); + + // Including AM Container, all other 4 containers of appC will be + // preempted + verify(mDisp, times(5)).handle(argThat(new IsPreemptionRequestFor(appC))); + + // By skipping AM Container, all other 4 containers of appB will be + // preempted + verify(mDisp, times(4)).handle(argThat(new IsPreemptionRequestFor(appB))); + + // By skipping AM Container, all other 4 containers of appA will be + // preempted + verify(mDisp, times(4)).handle(argThat(new IsPreemptionRequestFor(appA))); + setAMContainer = false; + } + static class IsPreemptionRequestFor extends ArgumentMatcher { private final ApplicationAttemptId appAttId; @@ -583,6 +686,9 @@ public int compare(FiCaSchedulerApp a1, FiCaSchedulerApp a2) { } } when(lq.getApplications()).thenReturn(qApps); + if(setAMResourcePercent != 0.0f){ + when(lq.getMaxAMResourcePerQueuePercent()).thenReturn(setAMResourcePercent); + } p.getChildQueues().add(lq); return lq; } @@ -607,7 +713,11 @@ FiCaSchedulerApp mockApp(int qid, int id, int used, int pending, int reserved, List cLive = new ArrayList(); for (int i = 0; i < used; i += gran) { - cLive.add(mockContainer(appAttId, cAlloc, unit, 1)); + if(setAMContainer && i == 0){ + cLive.add(mockContainer(appAttId, cAlloc, unit, 0)); + }else{ + cLive.add(mockContainer(appAttId, cAlloc, unit, 1)); + } ++cAlloc; } when(app.getLiveContainers()).thenReturn(cLive); @@ -623,6 +733,10 @@ RMContainer mockContainer(ApplicationAttemptId appAttId, int id, RMContainer mC = mock(RMContainer.class); when(mC.getContainerId()).thenReturn(cId); when(mC.getContainer()).thenReturn(c); + when(mC.getApplicationAttemptId()).thenReturn(appAttId); + if(0 == priority){ + when(mC.isAMContainer()).thenReturn(true); + } return mC; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index a4f173df190..b2d7c0687d0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -86,6 +86,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUnregistrationEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUpdateSavedEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; @@ -600,6 +602,9 @@ private Container allocateApplicationAttempt() { any(List.class), any(List.class))). thenReturn(allocation); + RMContainer rmContainer = mock(RMContainerImpl.class); + when(scheduler.getRMContainer(container.getId())). + thenReturn(rmContainer); applicationAttempt.handle( new RMAppAttemptContainerAllocatedEvent( From 58e398f31df9bd1c0fea4595f45a8731e877b7b1 Mon Sep 17 00:00:00 2001 From: Karthik Kambatla Date: Wed, 2 Jul 2014 02:05:37 +0000 Subject: [PATCH 077/112] YARN-2204. Explicitly enable vmem check in TestContainersMonitor#testContainerKillOnMemoryOverflow. (Anubhav Dhoot via kasha) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1607231 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 4 ++++ .../containermanager/monitor/TestContainersMonitor.java | 1 + 2 files changed, 5 insertions(+) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 55a8d16950e..c242a2918e5 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -211,6 +211,10 @@ Release 2.5.0 - UNRELEASED YARN-614. Changed ResourceManager to not count disk failure, node loss and RM restart towards app failures. (Xuan Gong via jianhe) + YARN-2204. Explicitly enable vmem check in + TestContainersMonitor#testContainerKillOnMemoryOverflow. + (Anubhav Dhoot via kasha) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java index 5a2fc2a7873..99d722f9c7a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java @@ -87,6 +87,7 @@ public void setup() throws IOException { conf.setClass( YarnConfiguration.NM_CONTAINER_MON_RESOURCE_CALCULATOR, LinuxResourceCalculatorPlugin.class, ResourceCalculatorPlugin.class); + conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, true); super.setup(); } From c89bf0184d56c34f09809851f7a8a1275bf5d359 Mon Sep 17 00:00:00 2001 From: Chris Nauroth Date: Wed, 2 Jul 2014 03:20:25 +0000 Subject: [PATCH 078/112] HDFS-6603. Add XAttr with ACL test. Contributed by Stephen Chu. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1607239 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../hdfs/server/namenode/FSXAttrBaseTest.java | 87 ++++++++++++++++++- .../server/namenode/TestFSImageWithXAttr.java | 10 ++- .../snapshot/TestXAttrWithSnapshot.java | 12 +++ .../TestOfflineImageViewer.java | 2 + 5 files changed, 110 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 1c424753be8..b62eb0a19f8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -482,6 +482,8 @@ Release 2.5.0 - UNRELEASED HDFS-6572. Add an option to the NameNode that prints the software and on-disk image versions. (Charles Lamb via cnauroth) + HDFS-6603. Add XAttr with ACL test. (Stephen Chu via cnauroth) + OPTIMIZATIONS HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java index 39679f86b5a..86f1ec90b8b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java @@ -32,12 +32,20 @@ import org.apache.hadoop.fs.XAttrSetFlag; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.test.GenericTestUtils; + +import static org.apache.hadoop.fs.permission.AclEntryScope.ACCESS; +import static org.apache.hadoop.fs.permission.AclEntryType.USER; +import static org.apache.hadoop.fs.permission.FsAction.ALL; +import static org.apache.hadoop.fs.permission.FsAction.READ; +import static org.apache.hadoop.hdfs.server.namenode.AclTestHelpers.aclEntry; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import org.junit.After; @@ -60,7 +68,7 @@ public class FSXAttrBaseTest { protected static MiniDFSCluster dfsCluster; protected static Configuration conf; private static int pathCount = 0; - private static Path path; + protected static Path path; // XAttrs protected static final String name1 = "user.a1"; @@ -73,10 +81,16 @@ public class FSXAttrBaseTest { protected FileSystem fs; + private static final UserGroupInformation BRUCE = + UserGroupInformation.createUserForTesting("bruce", new String[] { }); + private static final UserGroupInformation DIANA = + UserGroupInformation.createUserForTesting("diana", new String[] { }); + @BeforeClass public static void init() throws Exception { conf = new HdfsConfiguration(); conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY, true); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true); conf.setInt(DFSConfigKeys.DFS_NAMENODE_MAX_XATTRS_PER_INODE_KEY, 3); conf.setInt(DFSConfigKeys.DFS_NAMENODE_MAX_XATTR_SIZE_KEY, MAX_SIZE); initCluster(true); @@ -388,6 +402,21 @@ public void testRemoveXAttr() throws Exception { fs.removeXAttr(path, name3); } + @Test(timeout = 120000) + public void testRenameFileWithXAttr() throws Exception { + FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short)0750)); + fs.setXAttr(path, name1, value1, EnumSet.of(XAttrSetFlag.CREATE)); + fs.setXAttr(path, name2, value2, EnumSet.of(XAttrSetFlag.CREATE)); + Path renamePath = new Path(path.toString() + "-rename"); + fs.rename(path, renamePath); + Map xattrs = fs.getXAttrs(renamePath); + Assert.assertEquals(xattrs.size(), 2); + Assert.assertArrayEquals(value1, xattrs.get(name1)); + Assert.assertArrayEquals(value2, xattrs.get(name2)); + fs.removeXAttr(renamePath, name1); + fs.removeXAttr(renamePath, name2); + } + /** * Test the listXAttrs api. * listXAttrs on a path that doesn't exist. @@ -535,6 +564,50 @@ public void testCleanupXAttrs() throws Exception { Assert.assertArrayEquals(value1, xattrs.get(name1)); Assert.assertArrayEquals(value2, xattrs.get(name2)); } + + @Test(timeout = 120000) + public void testXAttrAcl() throws Exception { + FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) 0750)); + fs.setOwner(path, BRUCE.getUserName(), null); + FileSystem fsAsBruce = createFileSystem(BRUCE); + FileSystem fsAsDiana = createFileSystem(DIANA); + fsAsBruce.setXAttr(path, name1, value1); + + Map xattrs; + try { + xattrs = fsAsDiana.getXAttrs(path); + Assert.fail("Diana should not have read access to get xattrs"); + } catch (AccessControlException e) { + // Ignore + } + + // Give Diana read permissions to the path + fsAsBruce.modifyAclEntries(path, Lists.newArrayList( + aclEntry(ACCESS, USER, DIANA.getUserName(), READ))); + xattrs = fsAsDiana.getXAttrs(path); + Assert.assertArrayEquals(value1, xattrs.get(name1)); + + try { + fsAsDiana.removeXAttr(path, name1); + Assert.fail("Diana should not have write access to remove xattrs"); + } catch (AccessControlException e) { + // Ignore + } + + try { + fsAsDiana.setXAttr(path, name2, value2); + Assert.fail("Diana should not have write access to set xattrs"); + } catch (AccessControlException e) { + // Ignore + } + + fsAsBruce.modifyAclEntries(path, Lists.newArrayList( + aclEntry(ACCESS, USER, DIANA.getUserName(), ALL))); + fsAsDiana.setXAttr(path, name2, value2); + Assert.assertArrayEquals(value2, fsAsDiana.getXAttrs(path).get(name2)); + fsAsDiana.removeXAttr(path, name1); + fsAsDiana.removeXAttr(path, name2); + } /** * Creates a FileSystem for the super-user. @@ -545,6 +618,18 @@ public void testCleanupXAttrs() throws Exception { protected FileSystem createFileSystem() throws Exception { return dfsCluster.getFileSystem(); } + + /** + * Creates a FileSystem for a specific user. + * + * @param user UserGroupInformation specific user + * @return FileSystem for specific user + * @throws Exception if creation fails + */ + protected FileSystem createFileSystem(UserGroupInformation user) + throws Exception { + return DFSTestUtil.getFileSystemAs(user, conf); + } /** * Initializes all FileSystem instances used in the tests. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithXAttr.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithXAttr.java index 3240c5df2b9..bb6642f6155 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithXAttr.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithXAttr.java @@ -49,6 +49,8 @@ public class TestFSImageWithXAttr { private static final byte[] newValue1 = {0x31, 0x31, 0x31}; private static final String name2 = "user.a2"; private static final byte[] value2 = {0x37, 0x38, 0x39}; + private static final String name3 = "user.a3"; + private static final byte[] value3 = {}; @BeforeClass public static void setUp() throws IOException { @@ -70,25 +72,29 @@ private void testXAttr(boolean persistNamespace) throws IOException { fs.setXAttr(path, name1, value1, EnumSet.of(XAttrSetFlag.CREATE)); fs.setXAttr(path, name2, value2, EnumSet.of(XAttrSetFlag.CREATE)); + fs.setXAttr(path, name3, null, EnumSet.of(XAttrSetFlag.CREATE)); restart(fs, persistNamespace); Map xattrs = fs.getXAttrs(path); - Assert.assertEquals(xattrs.size(), 2); + Assert.assertEquals(xattrs.size(), 3); Assert.assertArrayEquals(value1, xattrs.get(name1)); Assert.assertArrayEquals(value2, xattrs.get(name2)); + Assert.assertArrayEquals(value3, xattrs.get(name3)); fs.setXAttr(path, name1, newValue1, EnumSet.of(XAttrSetFlag.REPLACE)); restart(fs, persistNamespace); xattrs = fs.getXAttrs(path); - Assert.assertEquals(xattrs.size(), 2); + Assert.assertEquals(xattrs.size(), 3); Assert.assertArrayEquals(newValue1, xattrs.get(name1)); Assert.assertArrayEquals(value2, xattrs.get(name2)); + Assert.assertArrayEquals(value3, xattrs.get(name3)); fs.removeXAttr(path, name1); fs.removeXAttr(path, name2); + fs.removeXAttr(path, name3); restart(fs, persistNamespace); xattrs = fs.getXAttrs(path); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestXAttrWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestXAttrWithSnapshot.java index 85c2fda126c..7042fc914ca 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestXAttrWithSnapshot.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestXAttrWithSnapshot.java @@ -268,6 +268,18 @@ public void testSetXAttrSnapshotPath() throws Exception { hdfs.setXAttr(snapshotPath, name1, value1); } + /** + * Assert exception of removing xattr on read-only snapshot. + */ + @Test + public void testRemoveXAttrSnapshotPath() throws Exception { + FileSystem.mkdirs(hdfs, path, FsPermission.createImmutable((short) 0700)); + hdfs.setXAttr(path, name1, value1); + SnapshotTestHelper.createSnapshot(hdfs, path, snapshotName); + exception.expect(SnapshotAccessControlException.class); + hdfs.removeXAttr(snapshotPath, name1); + } + /** * Assert exception of setting xattr when exceeding quota. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java index e5df79ec83f..c7e09ea9dc7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java @@ -143,6 +143,8 @@ public static void createOriginalFSImage() throws IOException { hdfs.mkdirs(xattr); hdfs.setXAttr(xattr, "user.a1", new byte[]{ 0x31, 0x32, 0x33 }); hdfs.setXAttr(xattr, "user.a2", new byte[]{ 0x37, 0x38, 0x39 }); + // OIV should be able to handle empty value XAttrs + hdfs.setXAttr(xattr, "user.a3", null); writtenFiles.put(xattr.toString(), hdfs.getFileStatus(xattr)); // Write results to the fsimage file From d1f54f4f4bed6c5f8e27bc80c3e33081f5c5d63f Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Wed, 2 Jul 2014 18:35:10 +0000 Subject: [PATCH 079/112] YARN-2065 AM cannot create new containers after restart git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1607441 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../ContainerManagerImpl.java | 22 +++++++------- .../server/TestContainerManagerSecurity.java | 30 +++++++++---------- 3 files changed, 27 insertions(+), 28 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index c242a2918e5..5c3de3d4c12 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -322,6 +322,9 @@ Release 2.5.0 - UNRELEASED YARN-2201. Made TestRMWebServicesAppsModification be independent of the changes on yarn-default.xml. (Varun Vasudev via zjshen) + YARN-2216 YARN-2065 AM cannot create new containers after restart + (Jian He via stevel) + Release 2.4.1 - 2014-06-23 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java index ded2013bfc9..1e155d27b84 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java @@ -475,8 +475,8 @@ protected void authorizeStartRequest(NMTokenIdentifier nmTokenIdentifier, boolean unauthorized = false; StringBuilder messageBuilder = new StringBuilder("Unauthorized request to start container. "); - if (!nmTokenIdentifier.getApplicationAttemptId().equals( - containerId.getApplicationAttemptId())) { + if (!nmTokenIdentifier.getApplicationAttemptId().getApplicationId().equals( + containerId.getApplicationAttemptId().getApplicationId())) { unauthorized = true; messageBuilder.append("\nNMToken for application attempt : ") .append(nmTokenIdentifier.getApplicationAttemptId()) @@ -810,26 +810,24 @@ protected void authorizeGetAndStopContainerRequest(ContainerId containerId, * belongs to the same application attempt (NMToken) which was used. (Note:- * This will prevent user in knowing another application's containers). */ - - if ((!identifier.getApplicationAttemptId().equals( - containerId.getApplicationAttemptId())) - || (container != null && !identifier.getApplicationAttemptId().equals( - container.getContainerId().getApplicationAttemptId()))) { + ApplicationId nmTokenAppId = + identifier.getApplicationAttemptId().getApplicationId(); + if ((!nmTokenAppId.equals(containerId.getApplicationAttemptId().getApplicationId())) + || (container != null && !nmTokenAppId.equals(container + .getContainerId().getApplicationAttemptId().getApplicationId()))) { if (stopRequest) { LOG.warn(identifier.getApplicationAttemptId() + " attempted to stop non-application container : " - + container.getContainerId().toString()); + + container.getContainerId()); NMAuditLogger.logFailure("UnknownUser", AuditConstants.STOP_CONTAINER, "ContainerManagerImpl", "Trying to stop unknown container!", - identifier.getApplicationAttemptId().getApplicationId(), - container.getContainerId()); + nmTokenAppId, container.getContainerId()); } else { LOG.warn(identifier.getApplicationAttemptId() + " attempted to get status for non-application container : " - + container.getContainerId().toString()); + + container.getContainerId()); } } - } class ContainerEventDispatcher implements EventHandler { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java index d607079235c..6797165dfe0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java @@ -202,8 +202,6 @@ private void testNMTokens(Configuration conf) throws Exception { ApplicationId appId = ApplicationId.newInstance(1, 1); ApplicationAttemptId validAppAttemptId = ApplicationAttemptId.newInstance(appId, 1); - ApplicationAttemptId invalidAppAttemptId = - ApplicationAttemptId.newInstance(appId, 2); ContainerId validContainerId = ContainerId.newInstance(validAppAttemptId, 0); @@ -269,26 +267,14 @@ private void testNMTokens(Configuration conf) throws Exception { testStartContainer(rpc, validAppAttemptId, validNode, validContainerToken, invalidNMToken, true))); - // using appAttempt-2 token for launching container for appAttempt-1. - invalidNMToken = - nmTokenSecretManagerRM.createNMToken(invalidAppAttemptId, validNode, - user); - sb = new StringBuilder("\nNMToken for application attempt : "); - sb.append(invalidAppAttemptId.toString()) - .append(" was used for starting container with container token") - .append(" issued for application attempt : ") - .append(validAppAttemptId.toString()); - Assert.assertTrue(testStartContainer(rpc, validAppAttemptId, validNode, - validContainerToken, invalidNMToken, true).contains(sb.toString())); - // using correct tokens. nmtoken for app attempt should get saved. conf.setInt(YarnConfiguration.RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS, 4 * 60 * 1000); validContainerToken = containerTokenSecretManager.createContainerToken(validContainerId, validNode, user, r, Priority.newInstance(0), 0); - testStartContainer(rpc, validAppAttemptId, validNode, validContainerToken, - validNMToken, false); + Assert.assertTrue(testStartContainer(rpc, validAppAttemptId, validNode, + validContainerToken, validNMToken, false).isEmpty()); Assert.assertTrue(nmTokenSecretManagerNM .isAppAttemptNMTokenKeyPresent(validAppAttemptId)); @@ -330,6 +316,18 @@ private void testNMTokens(Configuration conf) throws Exception { Assert.assertTrue(testGetContainer(rpc, validAppAttemptId, validNode, validContainerId, validNMToken, false).contains(sb.toString())); + // using appAttempt-1 NMtoken for launching container for appAttempt-2 should + // succeed. + ApplicationAttemptId attempt2 = ApplicationAttemptId.newInstance(appId, 2); + Token attempt1NMToken = + nmTokenSecretManagerRM + .createNMToken(validAppAttemptId, validNode, user); + org.apache.hadoop.yarn.api.records.Token newContainerToken = + containerTokenSecretManager.createContainerToken( + ContainerId.newInstance(attempt2, 1), validNode, user, r, + Priority.newInstance(0), 0); + Assert.assertTrue(testStartContainer(rpc, attempt2, validNode, + newContainerToken, attempt1NMToken, false).isEmpty()); } private void waitForContainerToFinishOnNM(ContainerId containerId) { From 9dfe9cf400abcdcd335e169113ca9c688e8db408 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Wed, 2 Jul 2014 18:37:00 +0000 Subject: [PATCH 080/112] HDFS-6612. MiniDFSNNTopology#simpleFederatedTopology(int) always hardcode nameservice ID. Contributed by Juan Yu. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1607442 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hadoop/hdfs/server/datanode/TestDeleteBlockPool.java | 3 ++- .../org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java | 3 ++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index b62eb0a19f8..8225872f4b8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -484,6 +484,9 @@ Release 2.5.0 - UNRELEASED HDFS-6603. Add XAttr with ACL test. (Stephen Chu via cnauroth) + HDFS-6612. MiniDFSNNTopology#simpleFederatedTopology(int) + always hardcode nameservice ID. (Juan Yu via wang) + OPTIMIZATIONS HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDeleteBlockPool.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDeleteBlockPool.java index cbe452dcee0..d16a4bb9e06 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDeleteBlockPool.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDeleteBlockPool.java @@ -160,7 +160,8 @@ public void testDfsAdminDeleteBlockPool() throws Exception { conf.set(DFSConfigKeys.DFS_NAMESERVICES, "namesServerId1,namesServerId2"); cluster = new MiniDFSCluster.Builder(conf) - .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(2)) + .nnTopology(MiniDFSNNTopology.simpleFederatedTopology( + conf.get(DFSConfigKeys.DFS_NAMESERVICES))) .numDataNodes(1).build(); cluster.waitActive(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java index 5061fe41456..34a314c6e3c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java @@ -1334,7 +1334,8 @@ public void testMultipleSecondaryNamenodes() throws IOException { SecondaryNameNode secondary2 = null; try { cluster = new MiniDFSCluster.Builder(conf) - .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(2)) + .nnTopology(MiniDFSNNTopology.simpleFederatedTopology( + conf.get(DFSConfigKeys.DFS_NAMESERVICES))) .build(); Configuration snConf1 = new HdfsConfiguration(cluster.getConfiguration(0)); Configuration snConf2 = new HdfsConfiguration(cluster.getConfiguration(1)); From 130182df1b5be514fa2399f8ed1e4f21dc3d132e Mon Sep 17 00:00:00 2001 From: Chris Nauroth Date: Wed, 2 Jul 2014 18:57:25 +0000 Subject: [PATCH 081/112] HDFS-6614. shorten TestPread run time with a smaller retry timeout setting. Contributed by Liang Xie. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1607447 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../src/test/java/org/apache/hadoop/hdfs/TestPread.java | 2 ++ 2 files changed, 5 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 8225872f4b8..ed484cd9e54 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -487,6 +487,9 @@ Release 2.5.0 - UNRELEASED HDFS-6612. MiniDFSNNTopology#simpleFederatedTopology(int) always hardcode nameservice ID. (Juan Yu via wang) + HDFS-6614. shorten TestPread run time with a smaller retry timeout setting. + (Liang Xie via cnauroth) + OPTIMIZATIONS HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java index 4fccc99d997..6b678154342 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java @@ -423,6 +423,8 @@ private void dfsPreadTest(Configuration conf, boolean disableTransferTo, boolean throws IOException { conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 4096); conf.setLong(DFSConfigKeys.DFS_CLIENT_READ_PREFETCH_SIZE_KEY, 4096); + // Set short retry timeouts so this test runs faster + conf.setInt(DFSConfigKeys.DFS_CLIENT_RETRY_WINDOW_BASE, 0); if (simulatedStorage) { SimulatedFSDataset.setFactory(conf); } From 297e3c72fb7369d32b1d76482dbff43eb0316edd Mon Sep 17 00:00:00 2001 From: Colin McCabe Date: Wed, 2 Jul 2014 19:43:36 +0000 Subject: [PATCH 082/112] HDFS-6604. The short-circuit cache doesn't correctly time out replicas that haven't been used in a while (cmccabe) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1607456 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hadoop/hdfs/shortcircuit/ShortCircuitCache.java | 8 ++------ .../hadoop/hdfs/shortcircuit/TestShortCircuitCache.java | 5 +++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index ed484cd9e54..3c81527eb46 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -750,6 +750,9 @@ Release 2.5.0 - UNRELEASED HDFS-6591. while loop is executed tens of thousands of times in Hedged Read (Liang Xie via cnauroth) + HDFS-6604. The short-circuit cache doesn't correctly time out replicas that + haven't been used in a while (cmccabe) + BREAKDOWN OF HDFS-2006 SUBTASKS AND RELATED JIRAS HDFS-6299. Protobuf for XAttr and client-side implementation. (Yi Liu via umamahesh) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java index 7df340ac2a6..a4b852f32e4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitCache.java @@ -111,7 +111,7 @@ public void run() { Long evictionTimeNs = Long.valueOf(0); while (true) { Entry entry = - evictableMmapped.ceilingEntry(evictionTimeNs); + evictable.ceilingEntry(evictionTimeNs); if (entry == null) break; evictionTimeNs = entry.getKey(); long evictionTimeMs = @@ -384,10 +384,6 @@ public ShortCircuitCache(int maxTotalSize, long maxNonMmappedEvictableLifespanMs this.shmManager = shmManager; } - public long getMmapRetryTimeoutMs() { - return mmapRetryTimeoutMs; - } - public long getStaleThresholdMs() { return staleThresholdMs; } @@ -847,7 +843,7 @@ ClientMmap getOrCreateClientMmap(ShortCircuitReplica replica, } else if (replica.mmapData instanceof Long) { long lastAttemptTimeMs = (Long)replica.mmapData; long delta = Time.monotonicNow() - lastAttemptTimeMs; - if (delta < staleThresholdMs) { + if (delta < mmapRetryTimeoutMs) { if (LOG.isTraceEnabled()) { LOG.trace(this + ": can't create client mmap for " + replica + " because we failed to " + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java index d62b3eb3201..a2d2bf830f3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java @@ -197,11 +197,12 @@ public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() { @Test(timeout=60000) public void testExpiry() throws Exception { final ShortCircuitCache cache = - new ShortCircuitCache(2, 1, 1, 10000000, 1, 10000, 0); + new ShortCircuitCache(2, 1, 1, 10000000, 1, 10000000, 0); final TestFileDescriptorPair pair = new TestFileDescriptorPair(); ShortCircuitReplicaInfo replicaInfo1 = cache.fetchOrCreate( - new ExtendedBlockId(123, "test_bp1"), new SimpleReplicaCreator(123, cache, pair)); + new ExtendedBlockId(123, "test_bp1"), + new SimpleReplicaCreator(123, cache, pair)); Preconditions.checkNotNull(replicaInfo1.getReplica()); Preconditions.checkState(replicaInfo1.getInvalidTokenException() == null); pair.compareWith(replicaInfo1.getReplica().getDataStream(), From 447c1c233bedc810aa3115f8306a64421c5522ed Mon Sep 17 00:00:00 2001 From: Karthik Kambatla Date: Wed, 2 Jul 2014 20:33:26 +0000 Subject: [PATCH 083/112] YARN-2241. ZKRMStateStore: On startup, show nicer messages if znodes already exist. (Robert Kanter via kasha) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1607473 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../recovery/ZKRMStateStore.java | 23 ++++++++++--------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 5c3de3d4c12..b9928a5e0a6 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -215,6 +215,9 @@ Release 2.5.0 - UNRELEASED TestContainersMonitor#testContainerKillOnMemoryOverflow. (Anubhav Dhoot via kasha) + YARN-2241. ZKRMStateStore: On startup, show nicer messages if znodes already + exist. (Robert Kanter via kasha) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java index 01bca39ad07..6dd4574b42e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java @@ -279,20 +279,21 @@ public synchronized void startInternal() throws Exception { private void createRootDir(final String rootPath) throws Exception { // For root dirs, we shouldn't use the doMulti helper methods - try { - new ZKAction() { - @Override - public String run() throws KeeperException, InterruptedException { + new ZKAction() { + @Override + public String run() throws KeeperException, InterruptedException { + try { return zkClient.create(rootPath, null, zkAcl, CreateMode.PERSISTENT); + } catch (KeeperException ke) { + if (ke.code() == Code.NODEEXISTS) { + LOG.debug(rootPath + "znode already exists!"); + return null; + } else { + throw ke; + } } - }.runWithRetries(); - } catch (KeeperException ke) { - if (ke.code() == Code.NODEEXISTS) { - LOG.debug(rootPath + "znode already exists!"); - } else { - throw ke; } - } + }.runWithRetries(); } private void logRootNodeAcls(String prefix) throws Exception { From 45b191e38cd226760f8e706ab0370747cd8f9321 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Wed, 2 Jul 2014 21:36:42 +0000 Subject: [PATCH 084/112] YARN-2232. Fixed ResourceManager to allow DelegationToken owners to be able to cancel their own tokens in secure mode. Contributed by Varun Vasudev. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1607484 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../resourcemanager/ClientRMService.java | 2 +- .../resourcemanager/TestClientRMService.java | 159 +++++++++++++++++- 3 files changed, 162 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index b9928a5e0a6..e3e6ada76cb 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -328,6 +328,9 @@ Release 2.5.0 - UNRELEASED YARN-2216 YARN-2065 AM cannot create new containers after restart (Jian He via stevel) + YARN-2232. Fixed ResourceManager to allow DelegationToken owners to be able + to cancel their own tokens in secure mode. (Varun Vasudev via vinodkv) + Release 2.4.1 - 2014-06-23 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index d8554bd2e1a..974376091b0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -919,7 +919,7 @@ public CancelDelegationTokenResponse cancelDelegationToken( protoToken.getIdentifier().array(), protoToken.getPassword().array(), new Text(protoToken.getKind()), new Text(protoToken.getService())); - String user = getRenewerForToken(token); + String user = UserGroupInformation.getCurrentUser().getUserName(); rmDTSecretManager.cancelToken(token, user); return Records.newRecord(CancelDelegationTokenResponse.class); } catch (IOException e) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 4b1f59c3039..4f4da37da1a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -44,16 +44,17 @@ import java.util.concurrent.CyclicBarrier; import org.junit.Assert; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.authentication.util.KerberosName; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.MockApps; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope; +import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest; @@ -138,6 +139,10 @@ public class TestClientRMService { private final static String QUEUE_1 = "Q-1"; private final static String QUEUE_2 = "Q-2"; + private final static String kerberosRule = "RULE:[1:$1@$0](.*@EXAMPLE.COM)s/@.*//\nDEFAULT"; + static { + KerberosName.setRules(kerberosRule); + } @BeforeClass public static void setupSecretManager() throws IOException { @@ -479,6 +484,17 @@ public void testGetQueueInfo() throws Exception { UserGroupInformation.createRemoteUser("owner"); private static final UserGroupInformation other = UserGroupInformation.createRemoteUser("other"); + private static final UserGroupInformation tester = + UserGroupInformation.createRemoteUser("tester"); + private static final String testerPrincipal = "tester@EXAMPLE.COM"; + private static final String ownerPrincipal = "owner@EXAMPLE.COM"; + private static final String otherPrincipal = "other@EXAMPLE.COM"; + private static final UserGroupInformation testerKerb = + UserGroupInformation.createRemoteUser(testerPrincipal); + private static final UserGroupInformation ownerKerb = + UserGroupInformation.createRemoteUser(ownerPrincipal); + private static final UserGroupInformation otherKerb = + UserGroupInformation.createRemoteUser(otherPrincipal); @Test public void testTokenRenewalByOwner() throws Exception { @@ -546,6 +562,147 @@ private void checkTokenRenewal(UserGroupInformation owner, rmService.renewDelegationToken(request); } + @Test + public void testTokenCancellationByOwner() throws Exception { + // two tests required - one with a kerberos name + // and with a short name + RMContext rmContext = mock(RMContext.class); + final ClientRMService rmService = + new ClientRMService(rmContext, null, null, null, null, dtsm); + testerKerb.doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + checkTokenCancellation(rmService, testerKerb, other); + return null; + } + }); + owner.doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + checkTokenCancellation(owner, other); + return null; + } + }); + } + + @Test + public void testTokenCancellationByRenewer() throws Exception { + // two tests required - one with a kerberos name + // and with a short name + RMContext rmContext = mock(RMContext.class); + final ClientRMService rmService = + new ClientRMService(rmContext, null, null, null, null, dtsm); + testerKerb.doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + checkTokenCancellation(rmService, owner, testerKerb); + return null; + } + }); + other.doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + checkTokenCancellation(owner, other); + return null; + } + }); + } + + @Test + public void testTokenCancellationByWrongUser() { + // two sets to test - + // 1. try to cancel tokens of short and kerberos users as a kerberos UGI + // 2. try to cancel tokens of short and kerberos users as a simple auth UGI + + RMContext rmContext = mock(RMContext.class); + final ClientRMService rmService = + new ClientRMService(rmContext, null, null, null, null, dtsm); + UserGroupInformation[] kerbTestOwners = + { owner, other, tester, ownerKerb, otherKerb }; + UserGroupInformation[] kerbTestRenewers = + { owner, other, ownerKerb, otherKerb }; + for (final UserGroupInformation tokOwner : kerbTestOwners) { + for (final UserGroupInformation tokRenewer : kerbTestRenewers) { + try { + testerKerb.doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + try { + checkTokenCancellation(rmService, tokOwner, tokRenewer); + Assert.fail("We should not reach here; token owner = " + + tokOwner.getUserName() + ", renewer = " + + tokRenewer.getUserName()); + return null; + } catch (YarnException e) { + Assert.assertTrue(e.getMessage().contains( + testerKerb.getUserName() + + " is not authorized to cancel the token")); + return null; + } + } + }); + } catch (Exception e) { + Assert.fail("Unexpected exception; " + e.getMessage()); + } + } + } + + UserGroupInformation[] simpleTestOwners = + { owner, other, ownerKerb, otherKerb, testerKerb }; + UserGroupInformation[] simpleTestRenewers = + { owner, other, ownerKerb, otherKerb }; + for (final UserGroupInformation tokOwner : simpleTestOwners) { + for (final UserGroupInformation tokRenewer : simpleTestRenewers) { + try { + tester.doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws Exception { + try { + checkTokenCancellation(tokOwner, tokRenewer); + Assert.fail("We should not reach here; token owner = " + + tokOwner.getUserName() + ", renewer = " + + tokRenewer.getUserName()); + return null; + } catch (YarnException ex) { + Assert.assertTrue(ex.getMessage().contains( + tester.getUserName() + + " is not authorized to cancel the token")); + return null; + } + } + }); + } catch (Exception e) { + Assert.fail("Unexpected exception; " + e.getMessage()); + } + } + } + } + + private void checkTokenCancellation(UserGroupInformation owner, + UserGroupInformation renewer) throws IOException, YarnException { + RMContext rmContext = mock(RMContext.class); + final ClientRMService rmService = + new ClientRMService(rmContext, null, null, null, null, dtsm); + checkTokenCancellation(rmService, owner, renewer); + } + + private void checkTokenCancellation(ClientRMService rmService, + UserGroupInformation owner, UserGroupInformation renewer) + throws IOException, YarnException { + RMDelegationTokenIdentifier tokenIdentifier = + new RMDelegationTokenIdentifier(new Text(owner.getUserName()), + new Text(renewer.getUserName()), null); + Token token = + new Token(tokenIdentifier, dtsm); + org.apache.hadoop.yarn.api.records.Token dToken = + BuilderUtils.newDelegationToken(token.getIdentifier(), token.getKind() + .toString(), token.getPassword(), token.getService().toString()); + CancelDelegationTokenRequest request = + Records.newRecord(CancelDelegationTokenRequest.class); + request.setDelegationToken(dToken); + rmService.cancelDelegationToken(request); + } + @Test (timeout = 30000) @SuppressWarnings ("rawtypes") public void testAppSubmit() throws Exception { From 304733679133aa804f85fd8f7d303207aefbf9af Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Wed, 2 Jul 2014 21:51:43 +0000 Subject: [PATCH 085/112] YARN-2022. Fixing CHANGES.txt to be correctly placed. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1607486 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index e3e6ada76cb..b503346fc2a 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -55,10 +55,6 @@ Release 2.5.0 - UNRELEASED (Varun Vasudev via vinodkv) IMPROVEMENTS - - YARN-2022 Preempting an Application Master container can be kept as least priority - when multiple applications are marked for preemption by - ProportionalCapacityPreemptionPolicy (Sunil G via mayank) YARN-1479. Invalid NaN values in Hadoop REST API JSON response (Chen He via jeagles) @@ -215,6 +211,10 @@ Release 2.5.0 - UNRELEASED TestContainersMonitor#testContainerKillOnMemoryOverflow. (Anubhav Dhoot via kasha) + YARN-2022. Preempting an Application Master container can be kept as least priority + when multiple applications are marked for preemption by + ProportionalCapacityPreemptionPolicy (Sunil G via mayank) + YARN-2241. ZKRMStateStore: On startup, show nicer messages if znodes already exist. (Robert Kanter via kasha) From 9b5d551ad94bd3343c3ca1689acaf33004425e02 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Wed, 2 Jul 2014 23:02:24 +0000 Subject: [PATCH 086/112] HDFS-6610. TestShortCircuitLocalRead tests sometimes timeout on slow machines. Contributed by Charles Lamb. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1607496 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../shortcircuit/TestShortCircuitLocalRead.java | 15 ++++++++------- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 3c81527eb46..1ca5354cad1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -490,6 +490,9 @@ Release 2.5.0 - UNRELEASED HDFS-6614. shorten TestPread run time with a smaller retry timeout setting. (Liang Xie via cnauroth) + HDFS-6610. TestShortCircuitLocalRead tests sometimes timeout on slow + machines. (Charles Lamb via wang) + OPTIMIZATIONS HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitLocalRead.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitLocalRead.java index bf27ffb6ef0..44eb79a1ad6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitLocalRead.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitLocalRead.java @@ -291,17 +291,17 @@ public void doTestShortCircuitReadImpl(boolean ignoreChecksum, int size, } } - @Test(timeout=10000) + @Test(timeout=60000) public void testFileLocalReadNoChecksum() throws Exception { doTestShortCircuitRead(true, 3*blockSize+100, 0); } - @Test(timeout=10000) + @Test(timeout=60000) public void testFileLocalReadChecksum() throws Exception { doTestShortCircuitRead(false, 3*blockSize+100, 0); } - @Test(timeout=10000) + @Test(timeout=60000) public void testSmallFileLocalRead() throws Exception { doTestShortCircuitRead(false, 13, 0); doTestShortCircuitRead(false, 13, 5); @@ -309,7 +309,7 @@ public void testSmallFileLocalRead() throws Exception { doTestShortCircuitRead(true, 13, 5); } - @Test(timeout=10000) + @Test(timeout=60000) public void testLocalReadLegacy() throws Exception { doTestShortCircuitReadLegacy(true, 13, 0, getCurrentUser(), getCurrentUser(), false); @@ -320,18 +320,18 @@ public void testLocalReadLegacy() throws Exception { * to use short circuit. The test ensures reader falls back to non * shortcircuit reads when shortcircuit is disallowed. */ - @Test(timeout=10000) + @Test(timeout=60000) public void testLocalReadFallback() throws Exception { doTestShortCircuitReadLegacy(true, 13, 0, getCurrentUser(), "notallowed", true); } - @Test(timeout=10000) + @Test(timeout=60000) public void testReadFromAnOffset() throws Exception { doTestShortCircuitRead(false, 3*blockSize+100, 777); doTestShortCircuitRead(true, 3*blockSize+100, 777); } - @Test(timeout=10000) + @Test(timeout=60000) public void testLongFile() throws Exception { doTestShortCircuitRead(false, 10*blockSize+100, 777); doTestShortCircuitRead(true, 10*blockSize+100, 777); @@ -578,6 +578,7 @@ public void run() { fs.delete(file1, false); } + @Test(timeout=60000) public void testReadWithRemoteBlockReader() throws IOException, InterruptedException { doTestShortCircuitReadWithRemoteBlockReader(true, 3*blockSize+100, getCurrentUser(), 0, false); } From c6a09d2110286632e6cfcee00abf8e79894381ec Mon Sep 17 00:00:00 2001 From: Zhijie Shen Date: Thu, 3 Jul 2014 01:43:56 +0000 Subject: [PATCH 087/112] MAPREDUCE-5900. Changed to the interpret container preemption exit code as a task attempt killing event. Contributed by Mayank Bansal. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1607512 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../v2/app/rm/RMContainerAllocator.java | 3 +- .../v2/app/job/impl/TestTaskAttempt.java | 173 ++++++++++++++++++ .../v2/app/rm/TestRMContainerAllocator.java | 16 ++ 4 files changed, 194 insertions(+), 1 deletion(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index e3a72b8046f..65c97b877f6 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -276,6 +276,9 @@ Release 2.5.0 - UNRELEASED MAPREDUCE-5939. StartTime showing up as the epoch time in JHS UI after upgrade (Chen He via jlowe) + MAPREDUCE-5900. Changed to the interpret container preemption exit code as a + task attempt killing event. (Mayank Bansal via zjshen) + Release 2.4.1 - 2014-06-23 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java index 11bc4063fff..64872cfe671 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java @@ -719,7 +719,8 @@ private List getResources() throws Exception { @VisibleForTesting public TaskAttemptEvent createContainerFinishedEvent(ContainerStatus cont, TaskAttemptId attemptID) { - if (cont.getExitStatus() == ContainerExitStatus.ABORTED) { + if (cont.getExitStatus() == ContainerExitStatus.ABORTED + || cont.getExitStatus() == ContainerExitStatus.PREEMPTED) { // killed by framework return new TaskAttemptEvent(attemptID, TaskAttemptEventType.TA_KILL); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java index 0cd0b000736..4aa11ed3ecf 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java @@ -65,6 +65,7 @@ import org.apache.hadoop.mapreduce.v2.app.job.Job; import org.apache.hadoop.mapreduce.v2.app.job.Task; import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt; +import org.apache.hadoop.mapreduce.v2.app.job.TaskAttemptStateInternal; import org.apache.hadoop.mapreduce.v2.app.job.event.JobEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.JobEventType; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptContainerAssignedEvent; @@ -795,6 +796,178 @@ public void testFetchFailureAttemptFinishTime() throws Exception{ finishTime, Long.valueOf(taImpl.getFinishTime())); } + @Test + public void testContainerKillAfterAssigned() throws Exception { + ApplicationId appId = ApplicationId.newInstance(1, 2); + ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, + 0); + JobId jobId = MRBuilderUtils.newJobId(appId, 1); + TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP); + TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0); + Path jobFile = mock(Path.class); + + MockEventHandler eventHandler = new MockEventHandler(); + TaskAttemptListener taListener = mock(TaskAttemptListener.class); + when(taListener.getAddress()).thenReturn( + new InetSocketAddress("localhost", 0)); + + JobConf jobConf = new JobConf(); + jobConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class); + jobConf.setBoolean("fs.file.impl.disable.cache", true); + jobConf.set(JobConf.MAPRED_MAP_TASK_ENV, ""); + jobConf.set(MRJobConfig.APPLICATION_ATTEMPT_ID, "10"); + + TaskSplitMetaInfo splits = mock(TaskSplitMetaInfo.class); + when(splits.getLocations()).thenReturn(new String[] { "127.0.0.1" }); + + AppContext appCtx = mock(AppContext.class); + ClusterInfo clusterInfo = mock(ClusterInfo.class); + Resource resource = mock(Resource.class); + when(appCtx.getClusterInfo()).thenReturn(clusterInfo); + when(resource.getMemory()).thenReturn(1024); + + TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler, + jobFile, 1, splits, jobConf, taListener, new Token(), + new Credentials(), new SystemClock(), appCtx); + + NodeId nid = NodeId.newInstance("127.0.0.2", 0); + ContainerId contId = ContainerId.newInstance(appAttemptId, 3); + Container container = mock(Container.class); + when(container.getId()).thenReturn(contId); + when(container.getNodeId()).thenReturn(nid); + when(container.getNodeHttpAddress()).thenReturn("localhost:0"); + + taImpl.handle(new TaskAttemptEvent(attemptId, + TaskAttemptEventType.TA_SCHEDULE)); + taImpl.handle(new TaskAttemptContainerAssignedEvent(attemptId, container, + mock(Map.class))); + assertEquals("Task attempt is not in assinged state", + taImpl.getInternalState(), TaskAttemptStateInternal.ASSIGNED); + taImpl.handle(new TaskAttemptEvent(attemptId, + TaskAttemptEventType.TA_KILL)); + assertEquals("Task should be in KILLED state", + TaskAttemptStateInternal.KILL_CONTAINER_CLEANUP, + taImpl.getInternalState()); + } + + @Test + public void testContainerKillWhileRunning() throws Exception { + ApplicationId appId = ApplicationId.newInstance(1, 2); + ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, + 0); + JobId jobId = MRBuilderUtils.newJobId(appId, 1); + TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP); + TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0); + Path jobFile = mock(Path.class); + + MockEventHandler eventHandler = new MockEventHandler(); + TaskAttemptListener taListener = mock(TaskAttemptListener.class); + when(taListener.getAddress()).thenReturn( + new InetSocketAddress("localhost", 0)); + + JobConf jobConf = new JobConf(); + jobConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class); + jobConf.setBoolean("fs.file.impl.disable.cache", true); + jobConf.set(JobConf.MAPRED_MAP_TASK_ENV, ""); + jobConf.set(MRJobConfig.APPLICATION_ATTEMPT_ID, "10"); + + TaskSplitMetaInfo splits = mock(TaskSplitMetaInfo.class); + when(splits.getLocations()).thenReturn(new String[] { "127.0.0.1" }); + + AppContext appCtx = mock(AppContext.class); + ClusterInfo clusterInfo = mock(ClusterInfo.class); + Resource resource = mock(Resource.class); + when(appCtx.getClusterInfo()).thenReturn(clusterInfo); + when(resource.getMemory()).thenReturn(1024); + + TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler, + jobFile, 1, splits, jobConf, taListener, new Token(), + new Credentials(), new SystemClock(), appCtx); + + NodeId nid = NodeId.newInstance("127.0.0.2", 0); + ContainerId contId = ContainerId.newInstance(appAttemptId, 3); + Container container = mock(Container.class); + when(container.getId()).thenReturn(contId); + when(container.getNodeId()).thenReturn(nid); + when(container.getNodeHttpAddress()).thenReturn("localhost:0"); + + taImpl.handle(new TaskAttemptEvent(attemptId, + TaskAttemptEventType.TA_SCHEDULE)); + taImpl.handle(new TaskAttemptContainerAssignedEvent(attemptId, container, + mock(Map.class))); + taImpl.handle(new TaskAttemptContainerLaunchedEvent(attemptId, 0)); + assertEquals("Task attempt is not in running state", taImpl.getState(), + TaskAttemptState.RUNNING); + taImpl.handle(new TaskAttemptEvent(attemptId, + TaskAttemptEventType.TA_KILL)); + assertFalse("InternalError occurred trying to handle TA_KILL", + eventHandler.internalError); + assertEquals("Task should be in KILLED state", + TaskAttemptStateInternal.KILL_CONTAINER_CLEANUP, + taImpl.getInternalState()); + } + + @Test + public void testContainerKillWhileCommitPending() throws Exception { + ApplicationId appId = ApplicationId.newInstance(1, 2); + ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, + 0); + JobId jobId = MRBuilderUtils.newJobId(appId, 1); + TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP); + TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0); + Path jobFile = mock(Path.class); + + MockEventHandler eventHandler = new MockEventHandler(); + TaskAttemptListener taListener = mock(TaskAttemptListener.class); + when(taListener.getAddress()).thenReturn( + new InetSocketAddress("localhost", 0)); + + JobConf jobConf = new JobConf(); + jobConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class); + jobConf.setBoolean("fs.file.impl.disable.cache", true); + jobConf.set(JobConf.MAPRED_MAP_TASK_ENV, ""); + jobConf.set(MRJobConfig.APPLICATION_ATTEMPT_ID, "10"); + + TaskSplitMetaInfo splits = mock(TaskSplitMetaInfo.class); + when(splits.getLocations()).thenReturn(new String[] { "127.0.0.1" }); + + AppContext appCtx = mock(AppContext.class); + ClusterInfo clusterInfo = mock(ClusterInfo.class); + Resource resource = mock(Resource.class); + when(appCtx.getClusterInfo()).thenReturn(clusterInfo); + when(resource.getMemory()).thenReturn(1024); + + TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler, + jobFile, 1, splits, jobConf, taListener, new Token(), + new Credentials(), new SystemClock(), appCtx); + + NodeId nid = NodeId.newInstance("127.0.0.2", 0); + ContainerId contId = ContainerId.newInstance(appAttemptId, 3); + Container container = mock(Container.class); + when(container.getId()).thenReturn(contId); + when(container.getNodeId()).thenReturn(nid); + when(container.getNodeHttpAddress()).thenReturn("localhost:0"); + + taImpl.handle(new TaskAttemptEvent(attemptId, + TaskAttemptEventType.TA_SCHEDULE)); + taImpl.handle(new TaskAttemptContainerAssignedEvent(attemptId, container, + mock(Map.class))); + taImpl.handle(new TaskAttemptContainerLaunchedEvent(attemptId, 0)); + assertEquals("Task attempt is not in running state", taImpl.getState(), + TaskAttemptState.RUNNING); + taImpl.handle(new TaskAttemptEvent(attemptId, + TaskAttemptEventType.TA_COMMIT_PENDING)); + assertEquals("Task should be in COMMIT_PENDING state", + TaskAttemptStateInternal.COMMIT_PENDING, taImpl.getInternalState()); + taImpl.handle(new TaskAttemptEvent(attemptId, + TaskAttemptEventType.TA_KILL)); + assertFalse("InternalError occurred trying to handle TA_KILL", + eventHandler.internalError); + assertEquals("Task should be in KILLED state", + TaskAttemptStateInternal.KILL_CONTAINER_CLEANUP, + taImpl.getInternalState()); + } + public static class MockEventHandler implements EventHandler { public boolean internalError; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java index 4c74d7b5c52..74edce22777 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java @@ -1959,6 +1959,22 @@ public void testCompletedContainerEvent() { TaskAttemptEvent abortedEvent = allocator.createContainerFinishedEvent( abortedStatus, attemptId); Assert.assertEquals(TaskAttemptEventType.TA_KILL, abortedEvent.getType()); + + ContainerId containerId2 = ContainerId.newInstance(applicationAttemptId, 2); + ContainerStatus status2 = ContainerStatus.newInstance(containerId2, + ContainerState.RUNNING, "", 0); + + ContainerStatus preemptedStatus = ContainerStatus.newInstance(containerId2, + ContainerState.RUNNING, "", ContainerExitStatus.PREEMPTED); + + TaskAttemptEvent event2 = allocator.createContainerFinishedEvent(status2, + attemptId); + Assert.assertEquals(TaskAttemptEventType.TA_CONTAINER_COMPLETED, + event2.getType()); + + TaskAttemptEvent abortedEvent2 = allocator.createContainerFinishedEvent( + preemptedStatus, attemptId); + Assert.assertEquals(TaskAttemptEventType.TA_KILL, abortedEvent2.getType()); } @Test From 54dc17ce2f911dddaf3217c9445acd5ba998354a Mon Sep 17 00:00:00 2001 From: Karthik Kambatla Date: Thu, 3 Jul 2014 05:55:05 +0000 Subject: [PATCH 088/112] Preparing for release 2.6.0 git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1607538 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 12 ++++++++++++ hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 12 ++++++++++++ hadoop-mapreduce-project/CHANGES.txt | 12 ++++++++++++ hadoop-yarn-project/CHANGES.txt | 12 ++++++++++++ 4 files changed, 48 insertions(+) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 0d65aa463a1..e7fa2e9159f 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -371,6 +371,18 @@ Trunk (Unreleased) HADOOP-8589. ViewFs tests fail when tests and home dirs are nested (sanjay Radia) +Release 2.6.0 - UNRELEASED + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + + BUG FIXES + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 1ca5354cad1..3681131f834 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -254,6 +254,18 @@ Trunk (Unreleased) HDFS-5794. Fix the inconsistency of layout version number of ADD_DATANODE_AND_STORAGE_UUIDS between trunk and branch-2. (jing9) +Release 2.6.0 - UNRELEASED + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + + BUG FIXES + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 65c97b877f6..37d69eb2fbd 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -145,6 +145,18 @@ Trunk (Unreleased) MAPREDUCE-5867. Fix NPE in KillAMPreemptionPolicy related to ProportionalCapacityPreemptionPolicy (Sunil G via devaraj) +Release 2.6.0 - UNRELEASED + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + + BUG FIXES + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index b503346fc2a..48af21190ac 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -18,6 +18,18 @@ Trunk - Unreleased YARN-2216 TestRMApplicationHistoryWriter sometimes fails in trunk. (Zhijie Shen via xgong) +Release 2.6.0 - UNRELEASED + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + + BUG FIXES + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES From 362ae5143c845e90ec27bf578084bf91947e3bca Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Thu, 3 Jul 2014 11:28:02 +0000 Subject: [PATCH 089/112] HADOOP-10312 Shell.ExitCodeException to have more useful toString git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1607591 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++ .../src/main/java/org/apache/hadoop/util/Shell.java | 12 +++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index e7fa2e9159f..f1cb98007ab 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -685,6 +685,8 @@ Release 2.5.0 - UNRELEASED HADOOP-10710. hadoop.auth cookie is not properly constructed according to RFC2109. (Juan Yu via tucu) + HADOOP-10312 Shell.ExitCodeException to have more useful toString (stevel) + Release 2.4.1 - 2014-06-23 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java index 69ac088fca9..fcdc021f4e4 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java @@ -621,7 +621,7 @@ public int getExitCode() { * This is an IOException with exit code added. */ public static class ExitCodeException extends IOException { - int exitCode; + private final int exitCode; public ExitCodeException(int exitCode, String message) { super(message); @@ -631,6 +631,16 @@ public ExitCodeException(int exitCode, String message) { public int getExitCode() { return exitCode; } + + @Override + public String toString() { + final StringBuilder sb = + new StringBuilder("ExitCodeException "); + sb.append("exitCode=").append(exitCode) + .append(": "); + sb.append(super.getMessage()); + return sb.toString(); + } } /** From e09ea0c06ee1caa5a9ebae0a8f0273dfe04d05e5 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Thu, 3 Jul 2014 12:04:50 +0000 Subject: [PATCH 090/112] HADOOP-9361: Strictly define FileSystem APIs git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1607596 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop/fs/BufferedFSInputStream.java | 11 +- .../apache/hadoop/fs/ChecksumFileSystem.java | 5 +- .../apache/hadoop/fs/FSDataOutputStream.java | 5 +- .../apache/hadoop/fs/FSExceptionMessages.java | 43 + .../org/apache/hadoop/fs/FSInputChecker.java | 5 +- .../apache/hadoop/fs/RawLocalFileSystem.java | 19 +- .../apache/hadoop/fs/ftp/FTPFileSystem.java | 119 ++- .../apache/hadoop/fs/ftp/FTPInputStream.java | 2 +- .../org/apache/hadoop/fs/s3/S3FileSystem.java | 3 +- .../s3native/Jets3tNativeFileSystemStore.java | 223 +++-- .../fs/s3native/NativeS3FileSystem.java | 99 ++- .../src/site/markdown/filesystem/extending.md | 95 +++ .../site/markdown/filesystem/filesystem.md | 802 ++++++++++++++++++ .../markdown/filesystem/fsdatainputstream.md | 379 +++++++++ .../src/site/markdown/filesystem/index.md | 37 + .../site/markdown/filesystem/introduction.md | 377 ++++++++ .../src/site/markdown/filesystem/model.md | 230 +++++ .../src/site/markdown/filesystem/notation.md | 191 +++++ .../src/site/markdown/filesystem/testing.md | 324 +++++++ .../apache/hadoop/fs/TestLocalFileSystem.java | 2 +- .../fs/contract/AbstractBondedFSContract.java | 115 +++ .../contract/AbstractContractAppendTest.java | 128 +++ .../contract/AbstractContractConcatTest.java | 112 +++ .../contract/AbstractContractCreateTest.java | 187 ++++ .../contract/AbstractContractDeleteTest.java | 97 +++ .../contract/AbstractContractMkdirTest.java | 115 +++ .../fs/contract/AbstractContractOpenTest.java | 155 ++++ .../contract/AbstractContractRenameTest.java | 185 ++++ .../AbstractContractRootDirectoryTest.java | 123 +++ .../fs/contract/AbstractContractSeekTest.java | 348 ++++++++ .../fs/contract/AbstractFSContract.java | 201 +++++ .../contract/AbstractFSContractTestBase.java | 363 ++++++++ .../hadoop/fs/contract/ContractOptions.java | 170 ++++ .../hadoop/fs/contract/ContractTestUtils.java | 759 +++++++++++++++++ .../hadoop/fs/contract/ftp/FTPContract.java | 63 ++ .../contract/ftp/TestFTPContractCreate.java | 32 + .../contract/ftp/TestFTPContractDelete.java | 32 + .../fs/contract/ftp/TestFTPContractMkdir.java | 34 + .../fs/contract/ftp/TestFTPContractOpen.java | 32 + .../contract/ftp/TestFTPContractRename.java | 66 ++ .../hadoop/fs/contract/ftp/package.html | 56 ++ .../fs/contract/localfs/LocalFSContract.java | 116 +++ .../localfs/TestLocalFSContractAppend.java | 32 + .../localfs/TestLocalFSContractCreate.java | 32 + .../localfs/TestLocalFSContractDelete.java | 32 + .../localfs/TestLocalFSContractLoaded.java | 53 ++ .../localfs/TestLocalFSContractMkdir.java | 34 + .../localfs/TestLocalFSContractOpen.java | 31 + .../localfs/TestLocalFSContractRename.java | 32 + .../localfs/TestLocalFSContractSeek.java | 31 + .../contract/rawlocal/RawlocalFSContract.java | 52 ++ ...awLocalContractUnderlyingFileBehavior.java | 49 ++ .../rawlocal/TestRawlocalContractAppend.java | 32 + .../rawlocal/TestRawlocalContractCreate.java | 32 + .../rawlocal/TestRawlocalContractDelete.java | 32 + .../rawlocal/TestRawlocalContractMkdir.java | 34 + .../rawlocal/TestRawlocalContractOpen.java | 31 + .../rawlocal/TestRawlocalContractRename.java | 32 + .../rawlocal/TestRawlocalContractSeek.java | 31 + .../fs/contract/s3n/NativeS3Contract.java | 43 + .../contract/s3n/TestS3NContractCreate.java | 38 + .../contract/s3n/TestS3NContractDelete.java | 31 + .../fs/contract/s3n/TestS3NContractMkdir.java | 34 + .../fs/contract/s3n/TestS3NContractOpen.java | 31 + .../contract/s3n/TestS3NContractRename.java | 32 + .../contract/s3n/TestS3NContractRootDir.java | 35 + .../fs/contract/s3n/TestS3NContractSeek.java | 31 + .../src/test/resources/contract/ftp.xml | 84 ++ .../src/test/resources/contract/localfs.xml | 110 +++ .../src/test/resources/contract/rawlocal.xml | 101 +++ .../src/test/resources/contract/s3n.xml | 95 +++ 71 files changed, 7569 insertions(+), 158 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSExceptionMessages.java create mode 100644 hadoop-common-project/hadoop-common/src/site/markdown/filesystem/extending.md create mode 100644 hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md create mode 100644 hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdatainputstream.md create mode 100644 hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md create mode 100644 hadoop-common-project/hadoop-common/src/site/markdown/filesystem/introduction.md create mode 100644 hadoop-common-project/hadoop-common/src/site/markdown/filesystem/model.md create mode 100644 hadoop-common-project/hadoop-common/src/site/markdown/filesystem/notation.md create mode 100644 hadoop-common-project/hadoop-common/src/site/markdown/filesystem/testing.md create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractBondedFSContract.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractAppendTest.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractConcatTest.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractCreateTest.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractDeleteTest.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMkdirTest.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractRenameTest.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractRootDirectoryTest.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSeekTest.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractFSContract.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractFSContractTestBase.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractOptions.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ftp/FTPContract.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ftp/TestFTPContractCreate.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ftp/TestFTPContractDelete.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ftp/TestFTPContractMkdir.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ftp/TestFTPContractOpen.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ftp/TestFTPContractRename.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ftp/package.html create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/LocalFSContract.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractAppend.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractCreate.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractDelete.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractLoaded.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractMkdir.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractOpen.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractRename.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractSeek.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/RawlocalFSContract.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawLocalContractUnderlyingFileBehavior.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractAppend.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractCreate.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractDelete.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractMkdir.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractOpen.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractRename.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractSeek.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/NativeS3Contract.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractCreate.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractDelete.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractMkdir.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractOpen.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRename.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRootDir.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractSeek.java create mode 100644 hadoop-common-project/hadoop-common/src/test/resources/contract/ftp.xml create mode 100644 hadoop-common-project/hadoop-common/src/test/resources/contract/localfs.xml create mode 100644 hadoop-common-project/hadoop-common/src/test/resources/contract/rawlocal.xml create mode 100644 hadoop-common-project/hadoop-common/src/test/resources/contract/s3n.xml diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BufferedFSInputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BufferedFSInputStream.java index be86ae84cd3..4855e0c3d2d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BufferedFSInputStream.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BufferedFSInputStream.java @@ -18,6 +18,7 @@ package org.apache.hadoop.fs; import java.io.BufferedInputStream; +import java.io.EOFException; import java.io.FileDescriptor; import java.io.IOException; @@ -51,6 +52,9 @@ public BufferedFSInputStream(FSInputStream in, int size) { @Override public long getPos() throws IOException { + if (in == null) { + throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED); + } return ((FSInputStream)in).getPos()-(count-pos); } @@ -66,8 +70,11 @@ public long skip(long n) throws IOException { @Override public void seek(long pos) throws IOException { - if( pos<0 ) { - return; + if (in == null) { + throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED); + } + if (pos < 0) { + throw new EOFException(FSExceptionMessages.NEGATIVE_SEEK); } if (this.pos != this.count) { // optimize: check if the pos is in the buffer diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java index 93cf60fed30..b67eadd94fc 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java @@ -18,6 +18,7 @@ package org.apache.hadoop.fs; +import java.io.EOFException; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; @@ -318,8 +319,8 @@ public synchronized long skip(long n) throws IOException { @Override public synchronized void seek(long pos) throws IOException { - if(pos>getFileLength()) { - throw new IOException("Cannot seek after EOF"); + if (pos > getFileLength()) { + throw new EOFException("Cannot seek after EOF"); } super.seek(pos); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStream.java index 689e9717d22..212fbbaa7a2 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStream.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStream.java @@ -67,7 +67,10 @@ long getPos() { @Override public void close() throws IOException { - out.close(); + // ensure close works even if a null reference was passed in + if (out != null) { + out.close(); + } } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSExceptionMessages.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSExceptionMessages.java new file mode 100644 index 00000000000..b80fb30f94b --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSExceptionMessages.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs; + +/** + * Standard strings to use in exception messages in filesystems + * HDFS is used as the reference source of the strings + */ +public class FSExceptionMessages { + + /** + * The operation failed because the stream is closed: {@value} + */ + public static final String STREAM_IS_CLOSED = "Stream is closed!"; + + /** + * Negative offset seek forbidden : {@value} + */ + public static final String NEGATIVE_SEEK = + "Cannot seek to a negative offset"; + + /** + * Seeks : {@value} + */ + public static final String CANNOT_SEEK_PAST_EOF = + "Attempted to seek or read past the end of the file"; +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSInputChecker.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSInputChecker.java index cc992e7c941..a88938e6f36 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSInputChecker.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSInputChecker.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.fs; +import java.io.EOFException; import java.io.IOException; import java.io.InputStream; import java.util.zip.Checksum; @@ -394,8 +395,8 @@ public synchronized long skip(long n) throws IOException { @Override public synchronized void seek(long pos) throws IOException { - if( pos<0 ) { - return; + if( pos < 0 ) { + throw new EOFException(FSExceptionMessages.NEGATIVE_SEEK); } // optimize: check if the pos is in the buffer long start = chunkPos - this.count; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java index 849b671c03a..a06e3a6eb24 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java @@ -23,6 +23,7 @@ import java.io.BufferedOutputStream; import java.io.DataOutput; +import java.io.EOFException; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -105,6 +106,10 @@ public LocalFSFileInputStream(Path f) throws IOException { @Override public void seek(long pos) throws IOException { + if (pos < 0) { + throw new EOFException( + FSExceptionMessages.NEGATIVE_SEEK); + } fis.getChannel().position(pos); this.position = pos; } @@ -256,7 +261,7 @@ private FSDataOutputStream create(Path f, boolean overwrite, boolean createParent, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { if (exists(f) && !overwrite) { - throw new IOException("File already exists: "+f); + throw new FileAlreadyExistsException("File already exists: " + f); } Path parent = f.getParent(); if (parent != null && !mkdirs(parent)) { @@ -272,7 +277,7 @@ public FSDataOutputStream createNonRecursive(Path f, FsPermission permission, EnumSet flags, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { if (exists(f) && !flags.contains(CreateFlag.OVERWRITE)) { - throw new IOException("File already exists: "+f); + throw new FileAlreadyExistsException("File already exists: " + f); } return new FSDataOutputStream(new BufferedOutputStream( new LocalFSFileOutputStream(f, false), bufferSize), statistics); @@ -344,6 +349,10 @@ public boolean rename(Path src, Path dst) throws IOException { @Override public boolean delete(Path p, boolean recursive) throws IOException { File f = pathToFile(p); + if (!f.exists()) { + //no path, return false "nothing to delete" + return false; + } if (f.isFile()) { return f.delete(); } else if (!recursive && f.isDirectory() && @@ -412,10 +421,14 @@ public boolean mkdirs(Path f) throws IOException { if(parent != null) { File parent2f = pathToFile(parent); if(parent2f != null && parent2f.exists() && !parent2f.isDirectory()) { - throw new FileAlreadyExistsException("Parent path is not a directory: " + throw new ParentNotDirectoryException("Parent path is not a directory: " + parent); } } + if (p2f.exists() && !p2f.isDirectory()) { + throw new FileNotFoundException("Destination exists" + + " and is not a directory: " + p2f.getCanonicalPath()); + } return (parent == null || mkdirs(parent)) && (p2f.mkdir() || p2f.isDirectory()); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java index 428bce3c9c6..9d36bcff8b0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java @@ -20,6 +20,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.net.ConnectException; import java.net.URI; import org.apache.commons.logging.Log; @@ -33,11 +34,14 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.ParentNotDirectoryException; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.util.Progressable; /** @@ -56,6 +60,12 @@ public class FTPFileSystem extends FileSystem { public static final int DEFAULT_BUFFER_SIZE = 1024 * 1024; public static final int DEFAULT_BLOCK_SIZE = 4 * 1024; + public static final String FS_FTP_USER_PREFIX = "fs.ftp.user."; + public static final String FS_FTP_HOST = "fs.ftp.host"; + public static final String FS_FTP_HOST_PORT = "fs.ftp.host.port"; + public static final String FS_FTP_PASSWORD_PREFIX = "fs.ftp.password."; + public static final String E_SAME_DIRECTORY_ONLY = + "only same directory renames are supported"; private URI uri; @@ -75,11 +85,11 @@ public void initialize(URI uri, Configuration conf) throws IOException { // get super.initialize(uri, conf); // get host information from uri (overrides info in conf) String host = uri.getHost(); - host = (host == null) ? conf.get("fs.ftp.host", null) : host; + host = (host == null) ? conf.get(FS_FTP_HOST, null) : host; if (host == null) { throw new IOException("Invalid host specified"); } - conf.set("fs.ftp.host", host); + conf.set(FS_FTP_HOST, host); // get port information from uri, (overrides info in conf) int port = uri.getPort(); @@ -96,11 +106,11 @@ public void initialize(URI uri, Configuration conf) throws IOException { // get } } String[] userPasswdInfo = userAndPassword.split(":"); - conf.set("fs.ftp.user." + host, userPasswdInfo[0]); + conf.set(FS_FTP_USER_PREFIX + host, userPasswdInfo[0]); if (userPasswdInfo.length > 1) { - conf.set("fs.ftp.password." + host, userPasswdInfo[1]); + conf.set(FS_FTP_PASSWORD_PREFIX + host, userPasswdInfo[1]); } else { - conf.set("fs.ftp.password." + host, null); + conf.set(FS_FTP_PASSWORD_PREFIX + host, null); } setConf(conf); this.uri = uri; @@ -115,23 +125,24 @@ public void initialize(URI uri, Configuration conf) throws IOException { // get private FTPClient connect() throws IOException { FTPClient client = null; Configuration conf = getConf(); - String host = conf.get("fs.ftp.host"); - int port = conf.getInt("fs.ftp.host.port", FTP.DEFAULT_PORT); - String user = conf.get("fs.ftp.user." + host); - String password = conf.get("fs.ftp.password." + host); + String host = conf.get(FS_FTP_HOST); + int port = conf.getInt(FS_FTP_HOST_PORT, FTP.DEFAULT_PORT); + String user = conf.get(FS_FTP_USER_PREFIX + host); + String password = conf.get(FS_FTP_PASSWORD_PREFIX + host); client = new FTPClient(); client.connect(host, port); int reply = client.getReplyCode(); if (!FTPReply.isPositiveCompletion(reply)) { - throw new IOException("Server - " + host - + " refused connection on port - " + port); + throw NetUtils.wrapException(host, port, + NetUtils.UNKNOWN_HOST, 0, + new ConnectException("Server response " + reply)); } else if (client.login(user, password)) { client.setFileTransferMode(FTP.BLOCK_TRANSFER_MODE); client.setFileType(FTP.BINARY_FILE_TYPE); client.setBufferSize(DEFAULT_BUFFER_SIZE); } else { throw new IOException("Login failed on server - " + host + ", port - " - + port); + + port + " as user '" + user + "'"); } return client; @@ -179,7 +190,7 @@ public FSDataInputStream open(Path file, int bufferSize) throws IOException { FileStatus fileStat = getFileStatus(client, absolute); if (fileStat.isDirectory()) { disconnect(client); - throw new IOException("Path " + file + " is a directory."); + throw new FileNotFoundException("Path " + file + " is a directory."); } client.allocate(bufferSize); Path parent = absolute.getParent(); @@ -214,12 +225,18 @@ public FSDataOutputStream create(Path file, FsPermission permission, final FTPClient client = connect(); Path workDir = new Path(client.printWorkingDirectory()); Path absolute = makeAbsolute(workDir, file); - if (exists(client, file)) { - if (overwrite) { - delete(client, file); + FileStatus status; + try { + status = getFileStatus(client, file); + } catch (FileNotFoundException fnfe) { + status = null; + } + if (status != null) { + if (overwrite && !status.isDirectory()) { + delete(client, file, false); } else { disconnect(client); - throw new IOException("File already exists: " + file); + throw new FileAlreadyExistsException("File already exists: " + file); } } @@ -272,14 +289,13 @@ public FSDataOutputStream append(Path f, int bufferSize, * Convenience method, so that we don't open a new connection when using this * method from within another method. Otherwise every API invocation incurs * the overhead of opening/closing a TCP connection. + * @throws IOException on IO problems other than FileNotFoundException */ - private boolean exists(FTPClient client, Path file) { + private boolean exists(FTPClient client, Path file) throws IOException { try { return getFileStatus(client, file) != null; } catch (FileNotFoundException fnfe) { return false; - } catch (IOException ioe) { - throw new FTPException("Failed to get file status", ioe); } } @@ -294,12 +310,6 @@ public boolean delete(Path file, boolean recursive) throws IOException { } } - /** @deprecated Use delete(Path, boolean) instead */ - @Deprecated - private boolean delete(FTPClient client, Path file) throws IOException { - return delete(client, file, false); - } - /** * Convenience method, so that we don't open a new connection when using this * method from within another method. Otherwise every API invocation incurs @@ -310,9 +320,14 @@ private boolean delete(FTPClient client, Path file, boolean recursive) Path workDir = new Path(client.printWorkingDirectory()); Path absolute = makeAbsolute(workDir, file); String pathName = absolute.toUri().getPath(); - FileStatus fileStat = getFileStatus(client, absolute); - if (fileStat.isFile()) { - return client.deleteFile(pathName); + try { + FileStatus fileStat = getFileStatus(client, absolute); + if (fileStat.isFile()) { + return client.deleteFile(pathName); + } + } catch (FileNotFoundException e) { + //the file is not there + return false; } FileStatus[] dirEntries = listStatus(client, absolute); if (dirEntries != null && dirEntries.length > 0 && !(recursive)) { @@ -491,7 +506,7 @@ private boolean mkdirs(FTPClient client, Path file, FsPermission permission) created = created && client.makeDirectory(pathName); } } else if (isFile(client, absolute)) { - throw new IOException(String.format( + throw new ParentNotDirectoryException(String.format( "Can't make directory for path %s since it is a file.", absolute)); } return created; @@ -527,6 +542,23 @@ public boolean rename(Path src, Path dst) throws IOException { } } + /** + * Probe for a path being a parent of another + * @param parent parent path + * @param child possible child path + * @return true if the parent's path matches the start of the child's + */ + private boolean isParentOf(Path parent, Path child) { + URI parentURI = parent.toUri(); + String parentPath = parentURI.getPath(); + if (!parentPath.endsWith("/")) { + parentPath += "/"; + } + URI childURI = child.toUri(); + String childPath = childURI.getPath(); + return childPath.startsWith(parentPath); + } + /** * Convenience method, so that we don't open a new connection when using this * method from within another method. Otherwise every API invocation incurs @@ -544,20 +576,31 @@ private boolean rename(FTPClient client, Path src, Path dst) Path absoluteSrc = makeAbsolute(workDir, src); Path absoluteDst = makeAbsolute(workDir, dst); if (!exists(client, absoluteSrc)) { - throw new IOException("Source path " + src + " does not exist"); + throw new FileNotFoundException("Source path " + src + " does not exist"); + } + if (isDirectory(absoluteDst)) { + // destination is a directory: rename goes underneath it with the + // source name + absoluteDst = new Path(absoluteDst, absoluteSrc.getName()); } if (exists(client, absoluteDst)) { - throw new IOException("Destination path " + dst - + " already exist, cannot rename!"); + throw new FileAlreadyExistsException("Destination path " + dst + + " already exists"); } String parentSrc = absoluteSrc.getParent().toUri().toString(); String parentDst = absoluteDst.getParent().toUri().toString(); - String from = src.getName(); - String to = dst.getName(); - if (!parentSrc.equals(parentDst)) { - throw new IOException("Cannot rename parent(source): " + parentSrc - + ", parent(destination): " + parentDst); + if (isParentOf(absoluteSrc, absoluteDst)) { + throw new IOException("Cannot rename " + absoluteSrc + " under itself" + + " : "+ absoluteDst); } + + if (!parentSrc.equals(parentDst)) { + throw new IOException("Cannot rename source: " + absoluteSrc + + " to " + absoluteDst + + " -"+ E_SAME_DIRECTORY_ONLY); + } + String from = absoluteSrc.getName(); + String to = absoluteDst.getName(); client.changeWorkingDirectory(parentSrc); boolean renamed = client.rename(from, to); return renamed; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPInputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPInputStream.java index 79dcfdb884c..22f3de2c544 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPInputStream.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPInputStream.java @@ -103,7 +103,7 @@ public synchronized int read(byte buf[], int off, int len) throws IOException { @Override public synchronized void close() throws IOException { if (closed) { - throw new IOException("Stream closed"); + return; } super.close(); closed = true; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java index 9240d3704ef..dda3cf683fa 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java @@ -32,6 +32,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -226,7 +227,7 @@ public FSDataOutputStream create(Path file, FsPermission permission, if (overwrite) { delete(file, true); } else { - throw new IOException("File already exists: " + file); + throw new FileAlreadyExistsException("File already exists: " + file); } } else { Path parent = file.getParent(); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java index 4618e8e67d5..a10d6f284fb 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java @@ -22,6 +22,7 @@ import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; +import java.io.EOFException; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -32,17 +33,19 @@ import java.util.Collections; import java.util.List; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSExceptionMessages; import org.apache.hadoop.fs.s3.S3Credentials; import org.apache.hadoop.fs.s3.S3Exception; +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.security.AccessControlException; import org.jets3t.service.S3Service; import org.jets3t.service.S3ServiceException; import org.jets3t.service.ServiceException; import org.jets3t.service.StorageObjectsChunk; +import org.jets3t.service.impl.rest.HttpException; import org.jets3t.service.impl.rest.httpclient.RestS3Service; import org.jets3t.service.model.MultipartPart; import org.jets3t.service.model.MultipartUpload; @@ -51,6 +54,8 @@ import org.jets3t.service.model.StorageObject; import org.jets3t.service.security.AWSCredentials; import org.jets3t.service.utils.MultipartUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @InterfaceAudience.Private @InterfaceStability.Unstable @@ -66,8 +71,8 @@ class Jets3tNativeFileSystemStore implements NativeFileSystemStore { private String serverSideEncryptionAlgorithm; - public static final Log LOG = - LogFactory.getLog(Jets3tNativeFileSystemStore.class); + public static final Logger LOG = + LoggerFactory.getLogger(Jets3tNativeFileSystemStore.class); @Override public void initialize(URI uri, Configuration conf) throws IOException { @@ -79,7 +84,7 @@ public void initialize(URI uri, Configuration conf) throws IOException { s3Credentials.getSecretAccessKey()); this.s3Service = new RestS3Service(awsCredentials); } catch (S3ServiceException e) { - handleS3ServiceException(e); + handleException(e); } multipartEnabled = conf.getBoolean("fs.s3n.multipart.uploads.enabled", false); @@ -115,16 +120,10 @@ public void storeFile(String key, File file, byte[] md5Hash) object.setMd5Hash(md5Hash); } s3Service.putObject(bucket, object); - } catch (S3ServiceException e) { - handleS3ServiceException(e); + } catch (ServiceException e) { + handleException(e, key); } finally { - if (in != null) { - try { - in.close(); - } catch (IOException e) { - // ignore - } - } + IOUtils.closeStream(in); } } @@ -147,10 +146,8 @@ public void storeLargeFile(String key, File file, byte[] md5Hash) try { mpUtils.uploadObjects(bucket.getName(), s3Service, objectsToUploadAsMultipart, null); - } catch (ServiceException e) { - handleServiceException(e); } catch (Exception e) { - throw new S3Exception(e); + handleException(e, key); } } @@ -163,8 +160,8 @@ public void storeEmptyFile(String key) throws IOException { object.setContentLength(0); object.setServerSideEncryptionAlgorithm(serverSideEncryptionAlgorithm); s3Service.putObject(bucket, object); - } catch (S3ServiceException e) { - handleS3ServiceException(e); + } catch (ServiceException e) { + handleException(e, key); } } @@ -172,20 +169,21 @@ public void storeEmptyFile(String key) throws IOException { public FileMetadata retrieveMetadata(String key) throws IOException { StorageObject object = null; try { - if(LOG.isDebugEnabled()) { - LOG.debug("Getting metadata for key: " + key + " from bucket:" + bucket.getName()); - } + LOG.debug("Getting metadata for key: {} from bucket: {}", + key, bucket.getName()); object = s3Service.getObjectDetails(bucket.getName(), key); return new FileMetadata(key, object.getContentLength(), object.getLastModifiedDate().getTime()); } catch (ServiceException e) { - // Following is brittle. Is there a better way? - if ("NoSuchKey".equals(e.getErrorCode())) { - return null; //return null if key not found + try { + // process + handleException(e, key); + return null; + } catch (FileNotFoundException fnfe) { + // and downgrade missing files + return null; } - handleServiceException(e); - return null; //never returned - keep compiler happy } finally { if (object != null) { object.closeDataInputStream(); @@ -204,13 +202,12 @@ public FileMetadata retrieveMetadata(String key) throws IOException { @Override public InputStream retrieve(String key) throws IOException { try { - if(LOG.isDebugEnabled()) { - LOG.debug("Getting key: " + key + " from bucket:" + bucket.getName()); - } + LOG.debug("Getting key: {} from bucket: {}", + key, bucket.getName()); S3Object object = s3Service.getObject(bucket.getName(), key); return object.getDataInputStream(); } catch (ServiceException e) { - handleServiceException(key, e); + handleException(e, key); return null; //return null if key not found } } @@ -228,15 +225,14 @@ public InputStream retrieve(String key) throws IOException { public InputStream retrieve(String key, long byteRangeStart) throws IOException { try { - if(LOG.isDebugEnabled()) { - LOG.debug("Getting key: " + key + " from bucket:" + bucket.getName() + " with byteRangeStart: " + byteRangeStart); - } + LOG.debug("Getting key: {} from bucket: {} with byteRangeStart: {}", + key, bucket.getName(), byteRangeStart); S3Object object = s3Service.getObject(bucket, key, null, null, null, null, byteRangeStart, null); return object.getDataInputStream(); } catch (ServiceException e) { - handleServiceException(key, e); - return null; //return null if key not found + handleException(e, key); + return null; } } @@ -254,17 +250,19 @@ public PartialListing list(String prefix, int maxListingLength, String priorLast } /** - * - * @return - * This method returns null if the list could not be populated - * due to S3 giving ServiceException - * @throws IOException + * list objects + * @param prefix prefix + * @param delimiter delimiter + * @param maxListingLength max no. of entries + * @param priorLastKey last key in any previous search + * @return a list of matches + * @throws IOException on any reported failure */ private PartialListing list(String prefix, String delimiter, int maxListingLength, String priorLastKey) throws IOException { try { - if (prefix.length() > 0 && !prefix.endsWith(PATH_DELIMITER)) { + if (!prefix.isEmpty() && !prefix.endsWith(PATH_DELIMITER)) { prefix += PATH_DELIMITER; } StorageObjectsChunk chunk = s3Service.listObjectsChunked(bucket.getName(), @@ -279,24 +277,20 @@ private PartialListing list(String prefix, String delimiter, } return new PartialListing(chunk.getPriorLastKey(), fileMetadata, chunk.getCommonPrefixes()); - } catch (S3ServiceException e) { - handleS3ServiceException(e); - return null; //never returned - keep compiler happy } catch (ServiceException e) { - handleServiceException(e); - return null; //return null if list could not be populated + handleException(e, prefix); + return null; // never returned - keep compiler happy } } @Override public void delete(String key) throws IOException { try { - if(LOG.isDebugEnabled()) { - LOG.debug("Deleting key:" + key + "from bucket" + bucket.getName()); - } + LOG.debug("Deleting key: {} from bucket: {}", + key, bucket.getName()); s3Service.deleteObject(bucket, key); } catch (ServiceException e) { - handleServiceException(key, e); + handleException(e, key); } } @@ -304,7 +298,7 @@ public void rename(String srcKey, String dstKey) throws IOException { try { s3Service.renameObject(bucket.getName(), srcKey, new S3Object(dstKey)); } catch (ServiceException e) { - handleServiceException(e); + handleException(e, srcKey); } } @@ -329,7 +323,7 @@ public void copy(String srcKey, String dstKey) throws IOException { s3Service.copyObject(bucket.getName(), srcKey, bucket.getName(), dstObject, false); } catch (ServiceException e) { - handleServiceException(srcKey, e); + handleException(e, srcKey); } } @@ -364,19 +358,22 @@ public void copyLargeFile(S3Object srcObject, String dstKey) throws IOException Collections.reverse(listedParts); s3Service.multipartCompleteUpload(multipartUpload, listedParts); } catch (ServiceException e) { - handleServiceException(e); + handleException(e, srcObject.getKey()); } } @Override public void purge(String prefix) throws IOException { + String key = ""; try { - S3Object[] objects = s3Service.listObjects(bucket.getName(), prefix, null); + S3Object[] objects = + s3Service.listObjects(bucket.getName(), prefix, null); for (S3Object object : objects) { - s3Service.deleteObject(bucket, object.getKey()); + key = object.getKey(); + s3Service.deleteObject(bucket, key); } } catch (S3ServiceException e) { - handleS3ServiceException(e); + handleException(e, key); } } @@ -390,39 +387,97 @@ public void dump() throws IOException { sb.append(object.getKey()).append("\n"); } } catch (S3ServiceException e) { - handleS3ServiceException(e); + handleException(e); } System.out.println(sb); } - private void handleServiceException(String key, ServiceException e) throws IOException { - if ("NoSuchKey".equals(e.getErrorCode())) { - throw new FileNotFoundException("Key '" + key + "' does not exist in S3"); + /** + * Handle any service exception by translating it into an IOException + * @param e exception + * @throws IOException exception -always + */ + private void handleException(Exception e) throws IOException { + throw processException(e, e, ""); + } + /** + * Handle any service exception by translating it into an IOException + * @param e exception + * @param key key sought from object store + + * @throws IOException exception -always + */ + private void handleException(Exception e, String key) throws IOException { + throw processException(e, e, key); + } + + /** + * Handle any service exception by translating it into an IOException + * @param thrown exception + * @param original original exception -thrown if no other translation could + * be made + * @param key key sought from object store or "" for undefined + * @return an exception to throw. If isProcessingCause==true this may be null. + */ + private IOException processException(Throwable thrown, Throwable original, + String key) { + IOException result; + if (thrown.getCause() != null) { + // recurse down + result = processException(thrown.getCause(), original, key); + } else if (thrown instanceof HttpException) { + // nested HttpException - examine error code and react + HttpException httpException = (HttpException) thrown; + String responseMessage = httpException.getResponseMessage(); + int responseCode = httpException.getResponseCode(); + String bucketName = "s3n://" + bucket.getName(); + String text = String.format("%s : %03d : %s", + bucketName, + responseCode, + responseMessage); + String filename = !key.isEmpty() ? (bucketName + "/" + key) : text; + IOException ioe; + switch (responseCode) { + case 404: + result = new FileNotFoundException(filename); + break; + case 416: // invalid range + result = new EOFException(FSExceptionMessages.CANNOT_SEEK_PAST_EOF + +": " + filename); + break; + case 403: //forbidden + result = new AccessControlException("Permission denied" + +": " + filename); + break; + default: + result = new IOException(text); + } + result.initCause(thrown); + } else if (thrown instanceof S3ServiceException) { + S3ServiceException se = (S3ServiceException) thrown; + LOG.debug( + "S3ServiceException: {}: {} : {}", + se.getS3ErrorCode(), se.getS3ErrorMessage(), se, se); + if ("InvalidRange".equals(se.getS3ErrorCode())) { + result = new EOFException(FSExceptionMessages.CANNOT_SEEK_PAST_EOF); + } else { + result = new S3Exception(se); + } + } else if (thrown instanceof ServiceException) { + ServiceException se = (ServiceException) thrown; + LOG.debug("S3ServiceException: {}: {} : {}", + se.getErrorCode(), se.toString(), se, se); + result = new S3Exception(se); + } else if (thrown instanceof IOException) { + result = (IOException) thrown; } else { - handleServiceException(e); + // here there is no exception derived yet. + // this means no inner cause, and no translation made yet. + // convert the original to an IOException -rather than just the + // exception at the base of the tree + result = new S3Exception(original); } - } - private void handleS3ServiceException(S3ServiceException e) throws IOException { - if (e.getCause() instanceof IOException) { - throw (IOException) e.getCause(); - } - else { - if(LOG.isDebugEnabled()) { - LOG.debug("S3 Error code: " + e.getS3ErrorCode() + "; S3 Error message: " + e.getS3ErrorMessage()); - } - throw new S3Exception(e); - } - } - - private void handleServiceException(ServiceException e) throws IOException { - if (e.getCause() instanceof IOException) { - throw (IOException) e.getCause(); - } - else { - if(LOG.isDebugEnabled()) { - LOG.debug("Got ServiceException with Error code: " + e.getErrorCode() + ";and Error message: " + e.getErrorMessage()); - } - } + return result; } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java index 7847ec5cc6c..e978e7067ef 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java @@ -19,6 +19,7 @@ package org.apache.hadoop.fs.s3native; import java.io.BufferedOutputStream; +import java.io.EOFException; import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; @@ -37,15 +38,16 @@ import java.util.TreeSet; import java.util.concurrent.TimeUnit; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; +import com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BufferedFSInputStream; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FSExceptionMessages; import org.apache.hadoop.fs.FSInputStream; +import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -55,6 +57,8 @@ import org.apache.hadoop.io.retry.RetryPolicy; import org.apache.hadoop.io.retry.RetryProxy; import org.apache.hadoop.util.Progressable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** *

    @@ -81,8 +85,8 @@ @InterfaceStability.Stable public class NativeS3FileSystem extends FileSystem { - public static final Log LOG = - LogFactory.getLog(NativeS3FileSystem.class); + public static final Logger LOG = + LoggerFactory.getLogger(NativeS3FileSystem.class); private static final String FOLDER_SUFFIX = "_$folder$"; static final String PATH_DELIMITER = Path.SEPARATOR; @@ -97,6 +101,7 @@ static class NativeS3FsInputStream extends FSInputStream { private long pos = 0; public NativeS3FsInputStream(NativeFileSystemStore store, Statistics statistics, InputStream in, String key) { + Preconditions.checkNotNull(in, "Null input stream"); this.store = store; this.statistics = statistics; this.in = in; @@ -105,13 +110,20 @@ public NativeS3FsInputStream(NativeFileSystemStore store, Statistics statistics, @Override public synchronized int read() throws IOException { - int result = -1; + int result; try { result = in.read(); } catch (IOException e) { - LOG.info("Received IOException while reading '" + key + "', attempting to reopen."); - seek(pos); - result = in.read(); + LOG.info("Received IOException while reading '{}', attempting to reopen", + key); + LOG.debug("{}", e, e); + try { + seek(pos); + result = in.read(); + } catch (EOFException eof) { + LOG.debug("EOF on input stream read: {}", eof, eof); + result = -1; + } } if (result != -1) { pos++; @@ -124,12 +136,17 @@ public synchronized int read() throws IOException { @Override public synchronized int read(byte[] b, int off, int len) throws IOException { - + if (in == null) { + throw new EOFException("Cannot read closed stream"); + } int result = -1; try { result = in.read(b, off, len); + } catch (EOFException eof) { + throw eof; } catch (IOException e) { - LOG.info("Received IOException while reading '" + key + "', attempting to reopen."); + LOG.info( "Received IOException while reading '{}'," + + " attempting to reopen.", key); seek(pos); result = in.read(b, off, len); } @@ -143,17 +160,53 @@ public synchronized int read(byte[] b, int off, int len) } @Override - public void close() throws IOException { - in.close(); + public synchronized void close() throws IOException { + closeInnerStream(); + } + + /** + * Close the inner stream if not null. Even if an exception + * is raised during the close, the field is set to null + * @throws IOException if raised by the close() operation. + */ + private void closeInnerStream() throws IOException { + if (in != null) { + try { + in.close(); + } finally { + in = null; + } + } + } + + /** + * Update inner stream with a new stream and position + * @param newStream new stream -must not be null + * @param newpos new position + * @throws IOException IO exception on a failure to close the existing + * stream. + */ + private synchronized void updateInnerStream(InputStream newStream, long newpos) throws IOException { + Preconditions.checkNotNull(newStream, "Null newstream argument"); + closeInnerStream(); + in = newStream; + this.pos = newpos; } @Override - public synchronized void seek(long pos) throws IOException { - in.close(); - LOG.info("Opening key '" + key + "' for reading at position '" + pos + "'"); - in = store.retrieve(key, pos); - this.pos = pos; + public synchronized void seek(long newpos) throws IOException { + if (newpos < 0) { + throw new EOFException( + FSExceptionMessages.NEGATIVE_SEEK); + } + if (pos != newpos) { + // the seek is attempting to move the current position + LOG.debug("Opening key '{}' for reading at position '{}", key, newpos); + InputStream newStream = store.retrieve(key, newpos); + updateInnerStream(newStream, newpos); + } } + @Override public synchronized long getPos() throws IOException { return pos; @@ -214,7 +267,7 @@ public synchronized void close() throws IOException { } backupStream.close(); - LOG.info("OutputStream for key '" + key + "' closed. Now beginning upload"); + LOG.info("OutputStream for key '{}' closed. Now beginning upload", key); try { byte[] md5Hash = digest == null ? null : digest.digest(); @@ -226,7 +279,7 @@ public synchronized void close() throws IOException { super.close(); closed = true; } - LOG.info("OutputStream for key '" + key + "' upload complete"); + LOG.info("OutputStream for key '{}' upload complete", key); } @Override @@ -339,7 +392,7 @@ public FSDataOutputStream create(Path f, FsPermission permission, Progressable progress) throws IOException { if (exists(f) && !overwrite) { - throw new IOException("File already exists:"+f); + throw new FileAlreadyExistsException("File already exists: " + f); } if(LOG.isDebugEnabled()) { @@ -367,7 +420,7 @@ public boolean delete(Path f, boolean recurse) throws IOException { String key = pathToKey(absolutePath); if (status.isDirectory()) { if (!recurse && listStatus(f).length > 0) { - throw new IOException("Can not delete " + f + " at is a not empty directory and recurse option is false"); + throw new IOException("Can not delete " + f + " as is a not empty directory and recurse option is false"); } createParent(f); @@ -538,7 +591,7 @@ private boolean mkdir(Path f) throws IOException { try { FileStatus fileStatus = getFileStatus(f); if (fileStatus.isFile()) { - throw new IOException(String.format( + throw new FileAlreadyExistsException(String.format( "Can't make directory for path '%s' since it is a file.", f)); } @@ -556,7 +609,7 @@ private boolean mkdir(Path f) throws IOException { public FSDataInputStream open(Path f, int bufferSize) throws IOException { FileStatus fs = getFileStatus(f); // will throw if the file doesn't exist if (fs.isDirectory()) { - throw new IOException("'" + f + "' is a directory"); + throw new FileNotFoundException("'" + f + "' is a directory"); } LOG.info("Opening '" + f + "' for reading"); Path absolutePath = makeAbsolute(f); diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/extending.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/extending.md new file mode 100644 index 00000000000..a09fea8e48b --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/extending.md @@ -0,0 +1,95 @@ + + +# Extending the File System specification and its tests + +The FileSystem specification is incomplete. It doesn't cover all operations or +even interfaces and classes in the FileSystem APIs. There may +be some minor issues with those that it does cover, such +as corner cases, failure modes, and other unexpected outcomes. It may also be that +a standard FileSystem significantly diverges from the specification, and +it is felt that this needs to be documented and coped with in tests. + +Finally, the FileSystem classes and methods are not fixed forever. +They may be extended with new operations on existing classes, as well as +potentially entirely new classes and interfaces. + +Accordingly, do not view this specification as a complete static document, +any more than the rest of the Hadoop code. + +1. View it as a live document to accompany the reference implementation (HDFS), +and the tests used to validate filesystems. +1. Don't be afraid to extend or correct it. +1. If you are proposing enhancements to the FileSystem APIs, you should extend the +specification to match. + +## How to update this specification + +1. Although found in the `hadoop-common` codebase, the HDFS team has ownership of +the FileSystem and FileContext APIs. Work with them on the hdfs-dev mailing list. + +1. Create JIRA issues in the `HADOOP` project, component `fs`, to cover changes +in the APIs and/or specification. + +1. Code changes will of course require tests. Ideally, changes to the specification +itself are accompanied by new tests. + +1. If the change involves operations that already have an `Abstract*ContractTest`, +add new test methods to the class and verify that they work on filesystem-specific +tests that subclass it. That includes the object stores as well as the local and +HDFS filesystems. + +1. If the changes add a new operation, add a new abstract test class +with the same contract-driven architecture as the existing one, and an implementation +subclass for all filesystems that support the operation. + +1. Add test methods to verify that invalid preconditions result in the expected +failures. + +1. Add test methods to verify that valid preconditions result in the expected +final state of the filesystem. Testing as little as possible per test aids +in tracking down problems. + +1. If possible, add tests to show concurrency expectations. + +If a FileSystem fails a newly added test, then it may be because: + +* The specification is wrong. +* The test is wrong. +* The test is looking for the wrong exception (i.e. it is too strict). +* The specification and tests are correct -and it is the filesystem is not +consistent with expectations. + +HDFS has to be treated as correct in its behavior. +If the test and specification do not match this behavior, then the specification +needs to be updated. Even so, there may be cases where the FS could be changed: + +1. The exception raised is a generic `IOException`, when a more informative +subclass, such as `EOFException` can be raised. +1. The FileSystem does not fail correctly when passed an invalid set of arguments. +This MAY be correctable, though must be done cautiously. + +If the mismatch is in LocalFileSystem, then it probably can't be corrected, as +this is the native filesystem as accessed via the Java IO APIs. + +For other FileSystems, their behaviour MAY be updated to more accurately reflect +the behavior of HDFS and/or LocalFileSystem. For most operations this is straightforward, +though the semantics of `rename()` are complicated enough that it is not clear +that HDFS is the correct reference. + +If a test fails and it is felt that it is a unfixable FileSystem-specific issue, then +a new contract option to allow for different interpretations of the results should +be added to the `ContractOptions` interface, the test modified to react to the +presence/absence of the option, and the XML contract files for the standard +FileSystems updated to indicate when a feature/failure mode is present. diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md new file mode 100644 index 00000000000..70796ccde12 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md @@ -0,0 +1,802 @@ + + + + + + + +# class `org.apache.hadoop.fs.FileSystem` + +The abstract `FileSystem` class is the original class to access Hadoop filesystems; +non-abstract subclasses exist for all Hadoop-supported filesystems. + +All operations that take a Path to this interface MUST support relative paths. +In such a case, they must be resolved relative to the working directory +defined by `setWorkingDirectory()`. + +For all clients, therefore, we also add the notion of a state component PWD: +this represents the present working directory of the client. Changes to this +state are not reflected in the filesystem itself: they are unique to the instance +of the client. + +**Implementation Note**: the static `FileSystem get(URI uri, Configuration conf) ` method MAY return +a pre-existing instance of a filesystem client class—a class that may also be in use in other threads. The implementations of `FileSystem` which ship with Apache Hadoop *do not make any attempt to synchronize access to the working directory field*. + +## Invariants + +All the requirements of a valid FileSystem are considered implicit preconditions and postconditions: +all operations on a valid FileSystem MUST result in a new FileSystem that is also valid. + + +## Predicates and other state access operations + + +### `boolean exists(Path p)` + + + def exists(FS, p) = p in paths(FS) + + +### `boolean isDirectory(Path p)` + + def isDirectory(FS, p)= p in directories(FS) + + +### `boolean isFile(Path p)` + + + def isFile(FS, p) = p in files(FS) + +### `boolean isSymlink(Path p)` + + + def isSymlink(FS, p) = p in symlinks(FS) + + +### `FileStatus getFileStatus(Path p)` + +Get the status of a path + +#### Preconditions + + + if not exists(FS, p) : raise FileNotFoundException + +#### Postconditions + + + result = stat: FileStatus where: + if isFile(FS, p) : + stat.length = len(FS.Files[p]) + stat.isdir = False + elif isDir(FS, p) : + stat.length = 0 + stat.isdir = True + elif isSymlink(FS, p) : + stat.length = 0 + stat.isdir = False + stat.symlink = FS.Symlinks[p] + +### `Path getHomeDirectory()` + +The function `getHomeDirectory` returns the home directory for the FileSystem +and the current user account. + +For some FileSystems, the path is `["/", "users", System.getProperty("user-name")]`. + +However, for HDFS, the username is derived from the credentials used to authenticate the client with HDFS. This +may differ from the local user account name. + +**It is the responsibility of the FileSystem to determine the actual home directory +of the caller.** + + +#### Preconditions + + +#### Postconditions + + result = p where valid-path(FS, p) + +There is no requirement that the path exists at the time the method was called, +or, if it exists, that it points to a directory. However, code tends to assume +that `not isFile(FS, getHomeDirectory())` holds to the extent that follow-on +code may fail. + +#### Implementation Notes + +* The FTPFileSystem queries this value from the remote filesystem and may +fail with a RuntimeException or subclass thereof if there is a connectivity +problem. The time to execute the operation is not bounded. + +### `FileSystem.listStatus(Path, PathFilter )` + +A `PathFilter` `f` is a predicate function that returns true iff the path `p` +meets the filter's conditions. + +#### Preconditions + +Path must exist: + + if not exists(FS, p) : raise FileNotFoundException + +#### Postconditions + + + if isFile(FS, p) and f(p) : + result = [getFileStatus(p)] + + elif isFile(FS, p) and not f(P) : + result = [] + + elif isDir(FS, p): + result [getFileStatus(c) for c in children(FS, p) where f(c) == True] + + +**Implicit invariant**: the contents of a `FileStatus` of a child retrieved +via `listStatus()` are equal to those from a call of `getFileStatus()` +to the same path: + + forall fs in listStatus(Path) : + fs == getFileStatus(fs.path) + + +### Atomicity and Consistency + +By the time the `listStatus()` operation returns to the caller, there +is no guarantee that the information contained in the response is current. +The details MAY be out of date, including the contents of any directory, the +attributes of any files, and the existence of the path supplied. + +The state of a directory MAY change during the evaluation +process. This may be reflected in a listing that is split between the pre- +and post-update FileSystem states. + + +* After an entry at path `P` is created, and before any other + changes are made to the FileSystem, `listStatus(P)` MUST + find the file and return its status. + +* After an entry at path `P` is deleted, `listStatus(P)` MUST + raise a `FileNotFoundException`. + +* After an entry at path `P` is created, and before any other + changes are made to the FileSystem, the result of `listStatus(parent(P))` SHOULD + include the value of `getFileStatus(P)`. + +* After an entry at path `P` is created, and before any other + changes are made to the FileSystem, the result of `listStatus(parent(P))` SHOULD + NOT include the value of `getFileStatus(P)`. + +This is not a theoretical possibility, it is observable in HDFS when a +directory contains many thousands of files. + +Consider a directory "d" with the contents: + + a + part-0000001 + part-0000002 + ... + part-9999999 + + +If the number of files is such that HDFS returns a partial listing in each +response, then, if a listing `listStatus("d")` takes place concurrently with the operation +`rename("d/a","d/z"))`, the result may be one of: + + [a, part-0000001, ... , part-9999999] + [part-0000001, ... , part-9999999, z] + + [a, part-0000001, ... , part-9999999, z] + [part-0000001, ... , part-9999999] + +While this situation is likely to be a rare occurrence, it MAY happen. In HDFS +these inconsistent views are only likely when listing a directory with many children. + +Other filesystems may have stronger consistency guarantees, or return inconsistent +data more readily. + +### ` List[BlockLocation] getFileBlockLocations(FileStatus f, int s, int l)` + +#### Preconditions + + if s < 0 or l < 0 : raise {HadoopIllegalArgumentException, InvalidArgumentException} + +* HDFS throws `HadoopIllegalArgumentException` for an invalid offset +or length; this extends `IllegalArgumentException`. + +#### Postconditions + +If the filesystem is location aware, it must return the list +of block locations where the data in the range `[s:s+l]` can be found. + + + if f == null : + result = null + elif f.getLen()) <= s + result = [] + else result = [ locations(FS, b) for all b in blocks(FS, p, s, s+l)] + +where + + def locations(FS, b) = a list of all locations of a block in the filesystem + + def blocks(FS, p, s, s + l) = a list of the blocks containing data(FS, path)[s:s+l] + + +Note that that as `length(FS, f) ` is defined as 0 if `isDir(FS, f)`, the result +of `getFileBlockLocations()` on a directory is [] + + +If the filesystem is not location aware, it SHOULD return + + [ + BlockLocation(["localhost:50010"] , + ["localhost"], + ["/default/localhost"] + 0, F.getLen()) + ] ; + + +*A bug in Hadoop 1.0.3 means that a topology path of the same number +of elements as the cluster topology MUST be provided, hence Filesystems SHOULD +return that `"/default/localhost"` path + + +### `getFileBlockLocations(Path P, int S, int L)` + +#### Preconditions + + + if p == null : raise NullPointerException + if not exists(FS, p) : raise FileNotFoundException + + +#### Postconditions + + result = getFileBlockLocations(getStatus(P), S, L) + + +### `getDefaultBlockSize()` + +#### Preconditions + +#### Postconditions + + result = integer >= 0 + +Although there is no defined minimum value for this result, as it +is used to partition work during job submission, a block size +that is too small will result in either too many jobs being submitted +for efficient work, or the `JobSubmissionClient` running out of memory. + + +Any FileSystem that does not actually break files into blocks SHOULD +return a number for this that results in efficient processing. +A FileSystem MAY make this user-configurable (the S3 and Swift filesystem clients do this). + +### `getDefaultBlockSize(Path P)` + +#### Preconditions + + +#### Postconditions + + + result = integer >= 0 + +The outcome of this operation is usually identical to `getDefaultBlockSize()`, +with no checks for the existence of the given path. + +Filesystems that support mount points may have different default values for +different paths, in which case the specific default value for the destination path +SHOULD be returned. + + +### `getBlockSize(Path P)` + +#### Preconditions + + if not exists(FS, p) : raise FileNotFoundException + + +#### Postconditions + + + result == getFileStatus(P).getBlockSize() + +The outcome of this operation MUST be identical to that contained in +the `FileStatus` returned from `getFileStatus(P)`. + + +## State Changing Operations + +### `boolean mkdirs(Path p, FsPermission permission )` + +Create a directory and all its parents + +#### Preconditions + + + if exists(FS, p) and not isDir(FS, p) : + raise [ParentNotDirectoryException, FileAlreadyExistsException, IOException] + + +#### Postconditions + + + FS' where FS'.Directories' = FS.Directories + [p] + ancestors(FS, p) + result = True + + +The condition exclusivity requirement of a FileSystem's directories, +files and symbolic links must hold. + +The probe for the existence and type of a path and directory creation MUST be +atomic. The combined operation, including `mkdirs(parent(F))` MAY be atomic. + +The return value is always true—even if a new directory is not created + (this is defined in HDFS). + +#### Implementation Notes: Local FileSystem + +The local FileSystem does not raise an exception if `mkdirs(p)` is invoked +on a path that exists and is a file. Instead the operation returns false. + + if isFile(FS, p): + FS' = FS + result = False + +### `FSDataOutputStream create(Path, ...)` + + + FSDataOutputStream create(Path p, + FsPermission permission, + boolean overwrite, + int bufferSize, + short replication, + long blockSize, + Progressable progress) throws IOException; + + +#### Preconditions + +The file must not exist for a no-overwrite create: + + if not overwrite and isFile(FS, p) : raise FileAlreadyExistsException + +Writing to or overwriting a directory must fail. + + if isDir(FS, p) : raise {FileAlreadyExistsException, FileNotFoundException, IOException} + + +FileSystems may reject the request for other +reasons, such as the FS being read-only (HDFS), +the block size being below the minimum permitted (HDFS), +the replication count being out of range (HDFS), +quotas on namespace or filesystem being exceeded, reserved +names, etc. All rejections SHOULD be `IOException` or a subclass thereof +and MAY be a `RuntimeException` or subclass. For instance, HDFS may raise a `InvalidPathException`. + +#### Postconditions + + FS' where : + FS'.Files'[p] == [] + ancestors(p) is-subset-of FS'.Directories' + + result = FSDataOutputStream + +The updated (valid) FileSystem must contains all the parent directories of the path, as created by `mkdirs(parent(p))`. + +The result is `FSDataOutputStream`, which through its operations may generate new filesystem states with updated values of +`FS.Files[p]` + +#### Implementation Notes + +* Some implementations split the create into a check for the file existing + from the + actual creation. This means the operation is NOT atomic: it is possible for + clients creating files with `overwrite==true` to fail if the file is created + by another client between the two tests. + +* S3N, Swift and potentially other Object Stores do not currently change the FS state +until the output stream `close()` operation is completed. +This MAY be a bug, as it allows >1 client to create a file with `overwrite==false`, + and potentially confuse file/directory logic + +* The Local FileSystem raises a `FileNotFoundException` when trying to create a file over +a directory, hence it is is listed as an exception that MAY be raised when +this precondition fails. + +* Not covered: symlinks. The resolved path of the symlink is used as the final path argument to the `create()` operation + +### `FSDataOutputStream append(Path p, int bufferSize, Progressable progress)` + +Implementations MAY throw `UnsupportedOperationException`. + +#### Preconditions + + if not exists(FS, p) : raise FileNotFoundException + + if not isFile(FS, p) : raise [FileNotFoundException, IOException] + +#### Postconditions + + FS + result = FSDataOutputStream + +Return: `FSDataOutputStream`, which can update the entry `FS.Files[p]` +by appending data to the existing list. + + +### `FSDataInputStream open(Path f, int bufferSize)` + +Implementations MAY throw `UnsupportedOperationException`. + +#### Preconditions + + if not isFile(FS, p)) : raise [FileNotFoundException, IOException] + +This is a critical precondition. Implementations of some FileSystems (e.g. +Object stores) could shortcut one round trip by postponing their HTTP GET +operation until the first `read()` on the returned `FSDataInputStream`. +However, much client code does depend on the existence check being performed +at the time of the `open()` operation. Implementations MUST check for the +presence of the file at the time of creation. This does not imply that +the file and its data is still at the time of the following `read()` or +any successors. + +#### Postconditions + + result = FSDataInputStream(0, FS.Files[p]) + +The result provides access to the byte array defined by `FS.Files[p]`; whether that +access is to the contents at the time the `open()` operation was invoked, +or whether and how it may pick up changes to that data in later states of FS is +an implementation detail. + +The result MUST be the same for local and remote callers of the operation. + + +#### HDFS implementation notes + +1. HDFS MAY throw `UnresolvedPathException` when attempting to traverse +symbolic links + +1. HDFS throws `IOException("Cannot open filename " + src)` if the path +exists in the metadata, but no copies of any its blocks can be located; +-`FileNotFoundException` would seem more accurate and useful. + + +### `FileSystem.delete(Path P, boolean recursive)` + +#### Preconditions + +A directory with children and recursive == false cannot be deleted + + if isDir(FS, p) and not recursive and (children(FS, p) != {}) : raise IOException + + +#### Postconditions + + +##### Nonexistent path + +If the file does not exist the FS state does not change + + if not exists(FS, p): + FS' = FS + result = False + +The result SHOULD be `False`, indicating that no file was deleted. + + +##### Simple File + + +A path referring to a file is removed, return value: `True` + + if isFile(FS, p) : + FS' = (FS.Directories, FS.Files - [p], FS.Symlinks) + result = True + + +##### Empty root directory + +Deleting an empty root does not change the filesystem state +and may return true or false. + + if isDir(FS, p) and isRoot(p) and children(FS, p) == {} : + FS ' = FS + result = (undetermined) + +There is no consistent return code from an attempt to delete the root directory. + +##### Empty (non-root) directory + +Deleting an empty directory that is not root will remove the path from the FS and +return true. + + if isDir(FS, p) and not isRoot(p) and children(FS, p) == {} : + FS' = (FS.Directories - [p], FS.Files, FS.Symlinks) + result = True + + +##### Recursive delete of root directory + +Deleting a root path with children and `recursive==True` + can do one of two things. + +The POSIX model assumes that if the user has +the correct permissions to delete everything, +they are free to do so (resulting in an empty filesystem). + + if isDir(FS, p) and isRoot(p) and recursive : + FS' = ({["/"]}, {}, {}, {}) + result = True + +In contrast, HDFS never permits the deletion of the root of a filesystem; the +filesystem can be taken offline and reformatted if an empty +filesystem is desired. + + if isDir(FS, p) and isRoot(p) and recursive : + FS' = FS + result = False + +##### Recursive delete of non-root directory + +Deleting a non-root path with children `recursive==true` +removes the path and all descendants + + if isDir(FS, p) and not isRoot(p) and recursive : + FS' where: + not isDir(FS', p) + and forall d in descendants(FS, p): + not isDir(FS', d) + not isFile(FS', d) + not isSymlink(FS', d) + result = True + +#### Atomicity + +* Deleting a file MUST be an atomic action. + +* Deleting an empty directory MUST be an atomic action. + +* A recursive delete of a directory tree MUST be atomic. + +#### Implementation Notes + +* S3N, Swift, FTP and potentially other non-traditional FileSystems +implement `delete()` as recursive listing and file delete operation. +This can break the expectations of client applications -and means that +they cannot be used as drop-in replacements for HDFS. + + + + + + +### `FileSystem.rename(Path src, Path d)` + +In terms of its specification, `rename()` is one of the most complex operations within a filesystem . + +In terms of its implementation, it is the one with the most ambiguity regarding when to return false +versus raising an exception. + +Rename includes the calculation of the destination path. +If the destination exists and is a directory, the final destination +of the rename becomes the destination + the filename of the source path. + + let dest = if (isDir(FS, src) and d != src) : + d + [filename(src)] + else : + d + +#### Preconditions + +All checks on the destination path MUST take place after the final `dest` path +has been calculated. + +Source `src` must exist: + + exists(FS, src) else raise FileNotFoundException + + +`dest` cannot be a descendant of `src`: + + if isDescendant(FS, src, dest) : raise IOException + +This implicitly covers the special case of `isRoot(FS, src)`. + +`dest` must be root, or have a parent that exists: + + isRoot(FS, dest) or exists(FS, parent(dest)) else raise IOException + +The parent path of a destination must not be a file: + + if isFile(FS, parent(dest)) : raise IOException + +This implicitly covers all the ancestors of the parent. + +There must not be an existing file at the end of the destination path: + + if isFile(FS, dest) : raise FileAlreadyExistsException, IOException + + +#### Postconditions + + +##### Renaming a directory onto itself + +Renaming a directory onto itself is no-op; return value is not specified. + +In POSIX the result is `False`; in HDFS the result is `True`. + + if isDir(FS, src) and src == dest : + FS' = FS + result = (undefined) + + +##### Renaming a file to self + +Renaming a file to itself is a no-op; the result is `True`. + + if isFile(FS, src) and src == dest : + FS' = FS + result = True + + +##### Renaming a file onto a nonexistent path + +Renaming a file where the destination is a directory moves the file as a child + of the destination directory, retaining the filename element of the source path. + + if isFile(FS, src) and src != dest: + FS' where: + not exists(FS', src) + and exists(FS', dest) + and data(FS', dest) == data (FS, dest) + result = True + + + +##### Renaming a directory onto a directory + +If `src` is a directory then all its children will then exist under `dest`, while the path +`src` and its descendants will no longer not exist. The names of the paths under +`dest` will match those under `src`, as will the contents: + + if isDir(FS, src) isDir(FS, dest) and src != dest : + FS' where: + not exists(FS', src) + and dest in FS'.Directories] + and forall c in descendants(FS, src) : + not exists(FS', c)) + and forall c in descendants(FS, src) where isDir(FS, c): + isDir(FS', dest + childElements(src, c) + and forall c in descendants(FS, src) where not isDir(FS, c): + data(FS', dest + childElements(s, c)) == data(FS, c) + result = True + +##### Renaming into a path where the parent path does not exist + + not exists(FS, parent(dest)) + +There is no consistent behavior here. + +*HDFS* + +The outcome is no change to FileSystem state, with a return value of false. + + FS' = FS; result = False + +*Local Filesystem, S3N* + +The outcome is as a normal rename, with the additional (implicit) feature +that the parent directores of the destination also exist + + exists(FS', parent(dest)) + +*Other Filesystems (including Swift) * + +Other filesystems strictly reject the operation, raising a `FileNotFoundException` + +##### Concurrency requirements + +* The core operation of `rename()`—moving one entry in the filesystem to +another—MUST be atomic. Some applications rely on this as a way to coordinate access to data. + +* Some FileSystem implementations perform checks on the destination +FileSystem before and after the rename. One example of this is `ChecksumFileSystem`, which +provides checksummed access to local data. The entire sequence MAY NOT be atomic. + +##### Implementation Notes + +**Files open for reading, writing or appending** + +The behavior of `rename()` on an open file is unspecified: whether it is +allowed, what happens to later attempts to read from or write to the open stream + +**Renaming a directory onto itself** + +The return code of renaming a directory onto itself is unspecified. + +**Destination exists and is a file** + +Renaming a file atop an existing file is specified as failing, raising an exception. + +* Local FileSystem : the rename succeeds; the destination file is replaced by the source file. + +* HDFS : The rename fails, no exception is raised. Instead the method call simply returns false. + +**Missing source file** + +If the source file `src` does not exist, `FileNotFoundException` should be raised. + +HDFS fails without raising an exception; `rename()` merely returns false. + + FS' = FS + result = false + +The behavior of HDFS here should not be considered a feature to replicate. +`FileContext` explicitly changed the behavior to raise an exception, and the retrofitting of that action +to the `DFSFileSystem` implementation is an ongoing matter for debate. + + +### `concat(Path p, Path sources[])` + +Joins multiple blocks together to create a single file. This +is a little-used operation currently implemented only by HDFS. + +Implementations MAY throw `UnsupportedOperationException` + +#### Preconditions + + if not exists(FS, p) : raise FileNotFoundException + + if sources==[] : raise IllegalArgumentException + +All sources MUST be in the same directory: + + for s in sources: if parent(S) != parent(p) raise IllegalArgumentException + +All block sizes must match that of the target: + + for s in sources: getBlockSize(FS, S) == getBlockSize(FS, p) + +No duplicate paths: + + not (exists p1, p2 in (sources + [p]) where p1 == p2) + +HDFS: All source files except the final one MUST be a complete block: + + for s in (sources[0:length(sources)-1] + [p]): + (length(FS, s) mod getBlockSize(FS, p)) == 0 + + +#### Postconditions + + + FS' where: + (data(FS', T) = data(FS, T) + data(FS, sources[0]) + ... + data(FS, srcs[length(srcs)-1])) + and for s in srcs: not exists(FS', S) + + +HDFS's restrictions may be an implementation detail of how it implements +`concat` -by changing the inode references to join them together in +a sequence. As no other filesystem in the Hadoop core codebase +implements this method, there is no way to distinguish implementation detail. +from specification. diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdatainputstream.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdatainputstream.md new file mode 100644 index 00000000000..0b0469bdedb --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdatainputstream.md @@ -0,0 +1,379 @@ + + + + + + + +# Class `FSDataInputStream extends DataInputStream` + +The core behavior of `FSDataInputStream` is defined by `java.io.DataInputStream`, +with extensions that add key assumptions to the system. + +1. The source is a local or remote filesystem. +1. The stream being read references a finite array of bytes. +1. The length of the data does not change during the read process. +1. The contents of the data does not change during the process. +1. The source file remains present during the read process +1. Callers may use `Seekable.seek()` to offsets within the array of bytes, with future +reads starting at this offset. +1. The cost of forward and backward seeks is low. +1. There is no requirement for the stream implementation to be thread-safe. + Callers MUST assume that instances are not thread-safe. + + +Files are opened via `FileSystem.open(p)`, which, if successful, returns: + + result = FSDataInputStream(0, FS.Files[p]) + +The stream can be modeled as: + + FSDIS = (pos, data[], isOpen) + +with access functions: + + pos(FSDIS) + data(FSDIS) + isOpen(FSDIS) + +**Implicit invariant**: the size of the data stream equals the size of the +file as returned by `FileSystem.getFileStatus(Path p)` + + forall p in dom(FS.Files[p]) : + len(data(FSDIS)) == FS.getFileStatus(p).length + + +### `Closeable.close()` + +The semantics of `java.io.Closeable` are defined in the interface definition +within the JRE. + +The operation MUST be idempotent; the following sequence is not an error: + + FSDIS.close(); + FSDIS.close(); + +#### Implementation Notes + +* Implementations SHOULD be robust against failure. If an inner stream +is closed, it should be checked for being `null` first. + +* Implementations SHOULD NOT raise `IOException` exceptions (or any other exception) +during this operation. Client applications often ignore these, or may fail +unexpectedly. + + + + + +#### Postconditions + + + FSDIS' = ((undefined), (undefined), False) + + +### `Seekable.getPos()` + +Return the current position. The outcome when a stream is closed is undefined. + +#### Preconditions + + isOpen(FSDIS) + +#### Postconditions + + result = pos(FSDIS) + + +### `InputStream.read()` + +Return the data at the current position. + +1. Implementations should fail when a stream is closed +1. There is no limit on how long `read()` may take to complete. + +#### Preconditions + + isOpen(FSDIS) + +#### Postconditions + + if ( pos < len(data) ): + FSDIS' = (pos + 1, data, True) + result = data[pos] + else + result = -1 + + +### `InputStream.read(buffer[], offset, length)` + +Read `length` bytes of data into the destination buffer, starting at offset +`offset` + +#### Preconditions + + isOpen(FSDIS) + buffer != null else raise NullPointerException + length >= 0 + offset < len(buffer) + length <= len(buffer) - offset + +Exceptions that may be raised on precondition failure are + + InvalidArgumentException + ArrayIndexOutOfBoundsException + RuntimeException + +#### Postconditions + + if length == 0 : + result = 0 + + elseif pos > len(data): + result -1 + + else + let l = min(length, len(data)-length) : + buffer' = buffer where forall i in [0..l-1]: + buffer'[o+i] = data[pos+i] + FSDIS' = (pos+l, data, true) + result = l + +### `Seekable.seek(s)` + + +#### Preconditions + +Not all subclasses implement the Seek operation: + + supported(FSDIS, Seekable.seek) else raise [UnsupportedOperationException, IOException] + +If the operation is supported, the file SHOULD be open: + + isOpen(FSDIS) + +Some filesystems do not perform this check, relying on the `read()` contract +to reject reads on a closed stream (e.g. `RawLocalFileSystem`). + +A `seek(0)` MUST always succeed, as the seek position must be +positive and less than the length of the Stream's: + + s > 0 and ((s==0) or ((s < len(data)))) else raise [EOFException, IOException] + +Some FileSystems do not raise an exception if this condition is not met. They +instead return -1 on any `read()` operation where, at the time of the read, +`len(data(FSDIS)) < pos(FSDIS)`. + +#### Postconditions + + FSDIS' = (s, data, True) + +There is an implicit invariant: a seek to the current position is a no-op + + seek(getPos()) + +Implementations may recognise this operation and bypass all other precondition +checks, leaving the input stream unchanged. + + +### `Seekable.seekToNewSource(offset)` + +This operation instructs the source to retrieve `data[]` from a different +source from the current source. This is only relevant if the filesystem supports +multiple replicas of a file and there is more than 1 replica of the +data at offset `offset`. + + +#### Preconditions + +Not all subclasses implement the operation operation, and instead +either raise an exception or return `False`. + + supported(FSDIS, Seekable.seekToNewSource) else raise [UnsupportedOperationException, IOException] + +Examples: `CompressionInputStream` , `HttpFSFileSystem` + +If supported, the file must be open: + + isOpen(FSDIS) + +#### Postconditions + +The majority of subclasses that do not implement this operation simply +fail. + + if not supported(FSDIS, Seekable.seekToNewSource(s)): + result = False + +Examples: `RawLocalFileSystem` , `HttpFSFileSystem` + +If the operation is supported and there is a new location for the data: + + FSDIS' = (pos, data', true) + result = True + +The new data is the original data (or an updated version of it, as covered +in the Consistency section below), but the block containing the data at `offset` +sourced from a different replica. + +If there is no other copy, `FSDIS` is not updated; the response indicates this: + + result = False + +Outside of test methods, the primary use of this method is in the {{FSInputChecker}} +class, which can react to a checksum error in a read by attempting to source +the data elsewhere. It a new source can be found it attempts to reread and +recheck that portion of the file. + +## interface `PositionedReadable` + +The `PositionedReadable` operations provide the ability to +read data into a buffer from a specific position in +the data stream. + +Although the interface declares that it must be thread safe, +some of the implementations do not follow this guarantee. + +#### Implementation preconditions + +Not all `FSDataInputStream` implementations support these operations. Those that do +not implement `Seekable.seek()` do not implement the `PositionedReadable` +interface. + + supported(FSDIS, Seekable.seek) else raise [UnsupportedOperationException, IOException] + +This could be considered obvious: if a stream is not Seekable, a client +cannot seek to a location. It is also a side effect of the +base class implementation, which uses `Seekable.seek()`. + + +**Implicit invariant**: for all `PositionedReadable` operations, the value +of `pos` is unchanged at the end of the operation + + pos(FSDIS') == pos(FSDIS) + + +There are no guarantees that this holds *during* the operation. + + +#### Failure states + +For any operations that fail, the contents of the destination +`buffer` are undefined. Implementations may overwrite part +or all of the buffer before reporting a failure. + + + +### `int PositionedReadable.read(position, buffer, offset, length)` + +#### Preconditions + + position > 0 else raise [IllegalArgumentException, RuntimeException] + len(buffer) + offset < len(data) else raise [IndexOutOfBoundException, RuntimeException] + length >= 0 + offset >= 0 + +#### Postconditions + +The amount of data read is the less of the length or the amount +of data available from the specified position: + + let available = min(length, len(data)-position) + buffer'[offset..(offset+available-1)] = data[position..position+available -1] + result = available + + +### `void PositionedReadable.readFully(position, buffer, offset, length)` + +#### Preconditions + + position > 0 else raise [IllegalArgumentException, RuntimeException] + length >= 0 + offset >= 0 + (position + length) <= len(data) else raise [EOFException, IOException] + len(buffer) + offset < len(data) + +#### Postconditions + +The amount of data read is the less of the length or the amount +of data available from the specified position: + + let available = min(length, len(data)-position) + buffer'[offset..(offset+length-1)] = data[position..(position + length -1)] + +### `PositionedReadable.readFully(position, buffer)` + +The semantics of this are exactly equivalent to + + readFully(position, buffer, 0, len(buffer)) + + +## Consistency + +* All readers, local and remote, of a data stream FSDIS provided from a `FileSystem.open(p)` +are expected to receive access to the data of `FS.Files[p]` at the time of opening. +* If the underlying data is changed during the read process, these changes MAY or +MAY NOT be visible. +* Such changes are visible MAY be partially visible. + + +At time t0 + + FSDIS0 = FS'read(p) = (0, data0[]) + +At time t1 + + FS' = FS' where FS'.Files[p] = data1 + +From time `t >= t1`, the value of `FSDIS0` is undefined. + +It may be unchanged + + FSDIS0.data == data0 + + forall l in len(FSDIS0.data): + FSDIS0.read() == data0[l] + + +It may pick up the new data + + FSDIS0.data == data1 + + forall l in len(FSDIS0.data): + FSDIS0.read() == data1[l] + +It may be inconsistent, such that a read of an offset returns +data from either of the datasets + + forall l in len(FSDIS0.data): + (FSDIS0.read(l) == data0[l]) or (FSDIS0.read(l) == data1[l])) + +That is, every value read may be from the original or updated file. + +It may also be inconsistent on repeated reads of same offset, that is +at time `t2 > t1`: + + r2 = FSDIS0.read(l) + +While at time `t3 > t2`: + + r3 = FSDIS0.read(l) + +It may be that `r3 != r2`. (That is, some of the data my be cached or replicated, +and on a subsequent read, a different version of the file's contents are returned). + + +Similarly, if the data at the path `p`, is deleted, this change MAY or MAY +not be visible during read operations performed on `FSDIS0`. diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md new file mode 100644 index 00000000000..66a7eb3f364 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md @@ -0,0 +1,37 @@ + + +# The Hadoop FileSystem API Definition + +This is a specification of the Hadoop FileSystem APIs, which models +the contents of a filesystem as a set of paths that are either directories, +symbolic links, or files. + +There is surprisingly little prior art in this area. There are multiple specifications of +Unix filesystems as a tree of inodes, but nothing public which defines the +notion of "Unix filesystem as a conceptual model for data storage access". + +This specification attempts to do that; to define the Hadoop FileSystem model +and APIs so that multiple filesystems can implement the APIs and present a consistent +model of their data to applications. It does not attempt to formally specify any of the +concurrency behaviors of the filesystems, other than to document the behaviours exhibited by +HDFS as these are commonly expected by Hadoop client applications. + +1. [Introduction](introduction.html) +1. [Notation](notation.html) +1. [Model](model.html) +1. [FileSystem class](filesystem.html) +1. [FSDataInputStream class](fsdatainputstream.html) +2. [Testing with the Filesystem specification](testing.html) +2. [Extending the specification and its tests](extending.html) diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/introduction.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/introduction.md new file mode 100644 index 00000000000..e451fa1f397 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/introduction.md @@ -0,0 +1,377 @@ + + +# Introduction + +This document defines the required behaviors of a Hadoop-compatible filesystem +for implementors and maintainers of the Hadoop filesystem, and for users of +the Hadoop FileSystem APIs + +Most of the Hadoop operations are tested against HDFS in the Hadoop test +suites, initially through `MiniDFSCluster`, before release by vendor-specific +'production' tests, and implicitly by the Hadoop stack above it. + +HDFS's actions have been modeled on POSIX filesystem behavior, using the actions and +return codes of Unix filesystem actions as a reference. Even so, there +are places where HDFS diverges from the expected behaviour of a POSIX +filesystem. + +The behaviour of other Hadoop filesystems are not as rigorously tested. +The bundled S3 FileSystem makes Amazon's S3 Object Store ("blobstore") +accessible through the FileSystem API. The Swift FileSystem driver provides similar +functionality for the OpenStack Swift blobstore. The Azure object storage +FileSystem in branch-1-win talks to Microsoft's Azure equivalent. All of these +bind to object stores, which do have different behaviors, especially regarding +consistency guarantees, and atomicity of operations. + +The "Local" FileSystem provides access to the underlying filesystem of the +platform. Its behavior is defined by the operating system and can +behave differently from HDFS. Examples of local filesystem quirks include +case-sensitivity, action when attempting to rename a file atop another file, +and whether it is possible to `seek()` past +the end of the file. + +There are also filesystems implemented by third parties that assert +compatibility with Apache Hadoop. There is no formal compatibility suite, and +hence no way for anyone to declare compatibility except in the form of their +own compatibility tests. + +These documents *do not* attempt to provide a normative definition of compatibility. +Passing the associated test suites *does not* guarantee correct behavior of applications. + +What the test suites do define is the expected set of actions—failing these +tests will highlight potential issues. + +By making each aspect of the contract tests configurable, it is possible to +declare how a filesystem diverges from parts of the standard contract. +This is information which can be conveyed to users of the filesystem. + +### Naming + +This document follows RFC 2119 rules regarding the use of MUST, MUST NOT, MAY, +and SHALL. MUST NOT is treated as normative. + +## Implicit assumptions of the Hadoop FileSystem APIs + +The original `FileSystem` class and its usages are based on an implicit set of +assumptions. Chiefly, that HDFS is +the underlying FileSystem, and that it offers a subset of the behavior of a +POSIX filesystem (or at least the implementation of the POSIX filesystem +APIs and model provided by Linux filesystems). + +Irrespective of the API, it's expected that all Hadoop-compatible filesystems +present the model of a filesystem implemented in Unix: + +* It's a hierarchical directory structure with files and directories. + +* Files contain zero or more bytes of data. + +* You cannot put files or directories under a file. + +* Directories contain zero or more files. + +* A directory entry has no data itself. + +* You can write arbitrary binary data to a file. When the file's contents + are read, from anywhere inside or outside of the cluster, the data is returned. + +* You can store many gigabytes of data in a single file. + +* The root directory, `"/"`, always exists, and cannot be renamed. + +* The root directory, `"/"`, is always a directory, and cannot be overwritten by a file write operation. + +* Any attempt to recursively delete the root directory will delete its contents (barring + lack of permissions), but will not delete the root path itself. + +* You cannot rename/move a directory under itself. + +* You cannot rename/move a directory atop any existing file other than the + source file itself. + +* Directory listings return all the data files in the directory (i.e. +there may be hidden checksum files, but all the data files are listed). + +* The attributes of a file in a directory listing (e.g. owner, length) match + the actual attributes of a file, and are consistent with the view from an + opened file reference. + +* Security: if the caller lacks the permissions for an operation, it will fail and raise an error. + +### Path Names + +* A Path is comprised of Path elements separated by `"/"`. + +* A path element is a unicode string of 1 or more characters. + +* Path element MUST NOT include the characters `":"` or `"/"`. + +* Path element SHOULD NOT include characters of ASCII/UTF-8 value 0-31 . + +* Path element MUST NOT be `"."` or `".."` + +* Note also that the Azure blob store documents say that paths SHOULD NOT use + a trailing `"."` (as their .NET URI class strips it). + + * Paths are compared based on unicode code-points. + + * Case-insensitive and locale-specific comparisons MUST NOT not be used. + +### Security Assumptions + +Except in the special section on security, this document assumes the client has +full access to the FileSystem. Accordingly, the majority of items in the list +do not add the qualification "assuming the user has the rights to perform the +operation with the supplied parameters and paths". + +The failure modes when a user lacks security permissions are not specified. + +### Networking Assumptions + +This document assumes this all network operations succeed. All statements +can be assumed to be qualified as *"assuming the operation does not fail due +to a network availability problem"* + +* The final state of a FileSystem after a network failure is undefined. + +* The immediate consistency state of a FileSystem after a network failure is undefined. + +* If a network failure can be reported to the client, the failure MUST be an +instance of `IOException` or subclass thereof. + +* The exception details SHOULD include diagnostics suitable for an experienced +Java developer _or_ operations team to begin diagnostics. For example, source +and destination hostnames and ports on a ConnectionRefused exception. + +* The exception details MAY include diagnostics suitable for inexperienced +developers to begin diagnostics. For example Hadoop tries to include a +reference to [ConnectionRefused](http://wiki.apache.org/hadoop/ConnectionRefused) when a TCP +connection request is refused. + + + +## Core Expectations of a Hadoop Compatible FileSystem + +Here are the core expectations of a Hadoop-compatible FileSystem. +Some FileSystems do not meet all these expectations; as a result, + some programs may not work as expected. + +### Atomicity + +There are some operations that MUST be atomic. This is because they are +often used to implement locking/exclusive access between processes in a cluster. + +1. Creating a file. If the `overwrite` parameter is false, the check and creation +MUST be atomic. +1. Deleting a file. +1. Renaming a file. +1. Renaming a directory. +1. Creating a single directory with `mkdir()`. + +* Recursive directory deletion MAY be atomic. Although HDFS offers atomic +recursive directory deletion, none of the other Hadoop FileSystems +offer such a guarantee (including local FileSystems). + +Most other operations come with no requirements or guarantees of atomicity. + + + +### Consistency + +The consistency model of a Hadoop FileSystem is *one-copy-update-semantics*; +that of a traditional local POSIX filesystem. Note that even NFS relaxes +some constraints about how fast changes propagate. + +* *Create.* Once the `close()` operation on an output stream writing a newly +created file has completed, in-cluster operations querying the file metadata +and contents MUST immediately see the file and its data. + +* *Update.* Once the `close()` operation on an output stream writing a newly +created file has completed, in-cluster operations querying the file metadata +and contents MUST immediately see the new data. + +* *Delete.* once a `delete()` operation on a path other than "/" has completed successfully, +it MUST NOT be visible or accessible. Specifically, +`listStatus()`, `open()` ,`rename()` and `append()` + operations MUST fail. + +* *Delete then create.* When a file is deleted then a new file of the same name created, the new file + MUST be immediately visible and its contents accessible via the FileSystem APIs. + +* *Rename.* After a `rename()` has completed, operations against the new path MUST +succeed; attempts to access the data against the old path MUST fail. + +* The consistency semantics inside of the cluster MUST be the same as outside of the cluster. +All clients querying a file that is not being actively manipulated MUST see the +same metadata and data irrespective of their location. + +### Concurrency + +There are no guarantees of isolated access to data: if one client is interacting +with a remote file and another client changes that file, the changes may or may +not be visible. + +### Operations and failures + +* All operations MUST eventually complete, successfully or unsuccessfully. + +* The time to complete an operation is undefined and may depend on +the implementation and on the state of the system. + +* Operations MAY throw a `RuntimeException` or subclass thereof. + +* Operations SHOULD raise all network, remote, and high-level problems as +an `IOException` or subclass thereof, and SHOULD NOT raise a +`RuntimeException` for such problems. + +* Operations SHOULD report failures by way of raised exceptions, rather +than specific return codes of an operation. + +* In the text, when an exception class is named, such as `IOException`, +the raised exception MAY be an instance or subclass of the named exception. +It MUST NOT be a superclass. + +* If an operation is not implemented in a class, the implementation must +throw an `UnsupportedOperationException`. + +* Implementations MAY retry failed operations until they succeed. If they do this, +they SHOULD do so in such a way that the *happens-before* relationship between +any sequence of operations meets the consistency and atomicity requirements +stated. See [HDFS-4849](https://issues.apache.org/jira/browse/HDFS-4849) +for an example of this: HDFS does not implement any retry feature that +could be observable by other callers. + +### Undefined capacity limits + +Here are some limits to FileSystem capacity that have never been explicitly +defined. + +1. The maximum number of files in a directory. + +1. Max number of directories in a directory + +1. Maximum total number of entries (files and directories) in a filesystem. + +1. The maximum length of a filename under a directory (HDFS: 8000). + +1. `MAX_PATH` - the total length of the entire directory tree referencing a +file. Blobstores tend to stop at ~1024 characters. + +1. The maximum depth of a path (HDFS: 1000 directories). + +1. The maximum size of a single file. + +### Undefined timeouts + +Timeouts for operations are not defined at all, including: + +* The maximum completion time of blocking FS operations. +MAPREDUCE-972 documents how `distcp` broke on slow s3 renames. + +* The timeout for idle read streams before they are closed. + +* The timeout for idle write streams before they are closed. + +The blocking-operation timeout is in fact variable in HDFS, as sites and +clients may tune the retry parameters so as to convert filesystem failures and +failovers into pauses in operation. Instead there is a general assumption that +FS operations are "fast but not as fast as local FS operations", and that the latency of data +reads and writes scale with the volume of data. This +assumption by client applications reveals a more fundamental one: that the filesystem is "close" +as far as network latency and bandwidth is concerned. + +There are also some implicit assumptions about the overhead of some operations. + +1. `seek()` operations are fast and incur little or no network delays. [This +does not hold on blob stores] + +1. Directory list operations are fast for directories with few entries. + +1. Directory list operations are fast for directories with few entries, but may +incur a cost that is `O(entries)`. Hadoop 2 added iterative listing to +handle the challenge of listing directories with millions of entries without +buffering -at the cost of consistency. + +1. A `close()` of an `OutputStream` is fast, irrespective of whether or not +the file operation has succeeded or not. + +1. The time to delete a directory is independent of the size of the number of +child entries + +### Object Stores vs. Filesystems + +This specification refers to *Object Stores* in places, often using the +term *Blobstore*. Hadoop does provide FileSystem client classes for some of these +even though they violate many of the requirements. This is why, although +Hadoop can read and write data in an object store, the two which Hadoop ships +with direct support for —Amazon S3 and OpenStack Swift&mdash cannot +be used as direct replacement for HDFS. + +*What is an Object Store?* + +An object store is a data storage service, usually accessed over HTTP/HTTPS. +A `PUT` request uploads an object/"Blob"; a `GET` request retrieves it; ranged +`GET` operations permit portions of a blob to retrieved. +To delete the object, the HTTP `DELETE` operation is invoked. + +Objects are stored by name: a string, possibly with "/" symbols in them. There +is no notion of a directory; arbitrary names can be assigned to objects — +within the limitations of the naming scheme imposed by the service's provider. + +The object stores invariably provide an operation to retrieve objects with +a given prefix; a `GET` operation on the root of the service with the +appropriate query parameters. + +Object stores usually prioritize availability —there is no single point +of failure equivalent to the HDFS NameNode(s). They also strive for simple +non-POSIX APIs: the HTTP verbs are the operations allowed. + +Hadoop FileSystem clients for object stores attempt to make the +stores pretend that they are a FileSystem, a FileSystem with the same +features and operations as HDFS. This is —ultimately—a pretence: +they have different characteristics and occasionally the illusion fails. + +1. **Consistency**. Object stores are generally *Eventually Consistent*: it +can take time for changes to objects —creation, deletion and updates— +to become visible to all callers. Indeed, there is no guarantee a change is +immediately visible to the client which just made the change. As an example, +an object `test/data1.csv` may be overwritten with a new set of data, but when +a `GET test/data1.csv` call is made shortly after the update, the original data +returned. Hadoop assumes that filesystems are consistent; that creation, updates +and deletions are immediately visible, and that the results of listing a directory +are current with respect to the files within that directory. + +1. **Atomicity**. Hadoop assumes that directory `rename()` operations are atomic, +as are `delete()` operations. Object store FileSystem clients implement these +as operations on the individual objects whose names match the directory prefix. +As a result, the changes take place a file at a time, and are not atomic. If +an operation fails part way through the process, the the state of the object store +reflects the partially completed operation. Note also that client code +assumes that these operations are `O(1)` —in an object store they are +more likely to be be `O(child-entries)`. + +1. **Durability**. Hadoop assumes that `OutputStream` implementations write data +to their (persistent) storage on a `flush()` operation. Object store implementations +save all their written data to a local file, a file that is then only `PUT` +to the object store in the final `close()` operation. As a result, there is +never any partial data from incomplete or failed operations. Furthermore, +as the write process only starts in `close()` operation, that operation may take +a time proportional to the quantity of data to upload, and inversely proportional +to the network bandwidth. It may also fail —a failure that is better +escalated than ignored. + +Object stores with these characteristics, can not be used as a direct replacement +for HDFS. In terms of this specification, their implementations of the +specified operations do not match those required. They are considered supported +by the Hadoop development community, but not to the same extent as HDFS. diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/model.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/model.md new file mode 100644 index 00000000000..d00dcd674b3 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/model.md @@ -0,0 +1,230 @@ + + +# A Model of a Hadoop Filesystem + + + +#### Paths and Path Elements + +A Path is a list of Path elements which represents a path to a file, directory of symbolic link + +Path elements are non-empty strings. The exact set of valid strings MAY +be specific to a particular FileSystem implementation. + +Path elements MUST NOT be in `{"", ".", "..", "/"}`. + +Path elements MUST NOT contain the characters `{'/', ':'}`. + +Filesystems MAY have other strings that are not permitted in a path element. + +When validating path elements, the exception `InvalidPathException` SHOULD +be raised when a path is invalid [HDFS] + +Predicate: `valid-path-element:List[String];` + +A path element `pe` is invalid if any character in it is in the set of forbidden characters, +or the element as a whole is invalid + + forall e in pe: not (e in {'/', ':'}) + not pe in {"", ".", "..", "/"} + + +Predicate: `valid-path:List` + +A Path `p` is *valid* if all path elements in it are valid + + def valid-path(pe): forall pe in Path: valid-path-element(pe) + + +The set of all possible paths is *Paths*; this is the infinite set of all lists of valid path elements. + +The path represented by empty list, `[]` is the *root path*, and is denoted by the string `"/"`. + +The partial function `parent(path:Path):Path` provides the parent path can be defined using +list slicing. + + def parent(pe) : pe[0:-1] + +Preconditions: + + path != [] + + +#### `filename:Path->PathElement` + +The last Path Element in a Path is called the filename. + + def filename(p) : p[-1] + +Preconditions: + + p != [] + +#### `childElements:(Path p, Path q):Path` + + +The partial function `childElements:(Path p, Path q):Path` +is the list of path elements in `p` that follow the path `q`. + + def childElements(p, q): p[len(q):] + +Preconditions: + + + # The path 'q' must be at the head of the path 'p' + q == p[:len(q)] + + +#### ancestors(Path): List[Path] + +The list of all paths that are either the direct parent of a path p, or a parent of +ancestor of p. + +#### Notes + +This definition handles absolute paths but not relative ones; it needs to be reworked so the root element is explicit, presumably +by declaring that the root (and only the root) path element may be ['/']. + +Relative paths can then be distinguished from absolute paths as the input to any function and resolved when the second entry in a two-argument function +such as `rename`. + +### Defining the Filesystem + + +A filesystem `FS` contains a set of directories, a dictionary of paths and a dictionary of symbolic links + + (Directories:set[Path], Files:[Path:List[byte]], Symlinks:set[Path]) + + +Accessor functions return the specific element of a filesystem + + def FS.Directories = FS.Directories + def file(FS) = FS.Files + def symlinks(FS) = FS.Symlinks + def filenames(FS) = keys(FS.Files) + +The entire set of a paths finite subset of all possible Paths, and functions to resolve a path to data, a directory predicate or a symbolic link: + + def paths(FS) = FS.Directories + filenames(FS) + FS.Symlinks) + +A path is deemed to exist if it is in this aggregate set: + + def exists(FS, p) = p in paths(FS) + +The root path, "/", is a directory represented by the path ["/"], which must always exist in a filesystem. + + def isRoot(p) = p == ["/"]. + + forall FS in FileSystems : ["/"] in FS.Directories + + + +#### Directory references + +A path MAY refer to a directory in a FileSystem: + + isDir(FS, p): p in FS.Directories + +Directories may have children, that is, there may exist other paths +in the FileSystem whose path begins with a directory. Only directories +may have children. This can be expressed +by saying that every path's parent must be a directory. + +It can then be declared that a path has no parent in which case it is the root directory, +or it MUST have a parent that is a directory: + + forall p in paths(FS) : isRoot(p) or isDir(FS, parent(p)) + +Because the parent directories of all directories must themselves satisfy +this criterion, it is implicit that only leaf nodes may be files or symbolic links: + +Furthermore, because every filesystem contains the root path, every filesystem +must contain at least one directory. + +A directory may have children: + + def children(FS, p) = {q for q in paths(FS) where parent(q) == p} + +There are no duplicate names in the child paths, because all paths are +taken from the set of lists of path elements. There can be no duplicate entries +in a set, hence no children with duplicate names. + +A path *D* is a descendant of a path *P* if it is the direct child of the +path *P* or an ancestor is a direct child of path *P*: + + def isDescendant(P, D) = parent(D) == P where isDescendant(P, parent(D)) + +The descendants of a directory P are all paths in the filesystem whose +path begins with the path P -that is their parent is P or an ancestor is P + + def descendants(FS, D) = {p for p in paths(FS) where isDescendant(D, p)} + + +#### File references + +A path MAY refer to a file; that it it has data in the filesystem; its path is a key in the data dictionary + + def isFile(FS, p) = p in FS.Files + + +#### Symbolic references + +A path MAY refer to a symbolic link: + + def isSymlink(FS, p) = p in symlinks(FS) + + +#### File Length + +The length of a path p in a filesystem FS is the length of the data stored, or 0 if it is a directory: + + def length(FS, p) = if isFile(p) : return length(data(FS, p)) else return 0 + +### User home + +The home directory of a user is an implicit part of a filesystem, and is derived from the userid of the +process working with the filesystem: + + def getHomeDirectory(FS) : Path + +The function `getHomeDirectory` returns the home directory for the Filesystem and the current user account. +For some FileSystems, the path is `["/","users", System.getProperty("user-name")]`. However, +for HDFS, + +#### Exclusivity + +A path cannot refer to more than one of a file, a directory or a symbolic link + + + FS.Directories ^ keys(data(FS)) == {} + FS.Directories ^ symlinks(FS) == {} + keys(data(FS))(FS) ^ symlinks(FS) == {} + + +This implies that only files may have data. + +This condition is invariant and is an implicit postcondition of all +operations that manipulate the state of a FileSystem `FS`. + +### Notes + +Not covered: hard links in a FileSystem. If a FileSystem supports multiple +references in *paths(FS)* to point to the same data, the outcome of operations +are undefined. + +This model of a FileSystem is sufficient to describe all the FileSystem +queries and manipulations excluding metadata and permission operations. +The Hadoop `FileSystem` and `FileContext` interfaces can be specified +in terms of operations that query or change the state of a FileSystem. diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/notation.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/notation.md new file mode 100644 index 00000000000..2b5ece8785a --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/notation.md @@ -0,0 +1,191 @@ + + + +# Notation + +A formal notation such as [The Z Notation](http://www.open-std.org/jtc1/sc22/open/n3187.pdf) +would be the strictest way to define Hadoop FileSystem behavior, and could even +be used to prove some axioms. + +However, it has a number of practical flaws: + +1. Such notations are not as widely used as they should be, so the broader software +development community is not going to have practical experience of it. + +1. It's very hard to work with without dropping into tools such as LaTeX *and* add-on libraries. + +1. Such notations are difficult to understand, even for experts. + +Given that the target audience of this specification is FileSystem developers, +formal notations are not appropriate. Instead, broad comprehensibility, ease of maintenance, and +ease of deriving tests take priority over mathematically-pure formal notation. + +### Mathematics Symbols in this document + +This document does use a subset of [the notation in the Z syntax](http://staff.washington.edu/jon/z/glossary.html), +but in an ASCII form and the use of Python list notation for manipulating lists and sets. + +* `iff` : `iff` If and only if +* `⇒` : `implies` +* `→` : `-->` total function +* `↛` : `->` partial function + + +* `∩` : `^`: Set Intersection +* `∪` : `+`: Set Union +* `\` : `-`: Set Difference + +* `∃` : `exists` Exists predicate +* `∀` : `forall`: For all predicate +* `=` : `==` Equals operator +* `≠` : `!=` operator. In Java `z ≠ y` is written as `!( z.equals(y))` for all non-simple datatypes +* `≡` : `equivalent-to` equivalence operator. This is stricter than equals. +* `∅` : `{}` Empty Set. `∅ ≡ {}` +* `≈` : `approximately-equal-to` operator +* `¬` : `not` Not operator. In Java, `!` +* `∄` : `does-not-exist`: Does not exist predicate. Equivalent to `not exists` +* `∧` : `and` : local and operator. In Java , `&&` +* `∨` : `or` : local and operator. In Java, `||` +* `∈` : `in` : element of +* `∉` : `not in` : not an element of +* `⊆` : `subset-or-equal-to` the subset or equality condition +* `⊂` : `subset-of` the proper subset condition +* `| p |` : `len(p)` the size of a variable + +* `:=` : `=` : + +* `` : `#` : Python-style comments + +* `happens-before` : `happens-before` : Lamport's ordering relationship as defined in +[Time, Clocks and the Ordering of Events in a Distributed System](http://research.microsoft.com/en-us/um/people/lamport/pubs/time-clocks.pdf) + +#### Sets, Lists, Maps, and Strings + +The [python data structures](http://docs.python.org/2/tutorial/datastructures.html) +are used as the basis for this syntax as it is both plain ASCII and well-known. + +##### Lists + +* A list *L* is an ordered sequence of elements `[e1, e2, ... en]` +* The size of a list `len(L)` is the number of elements in a list. +* Items can be addressed by a 0-based index `e1 == L[0]` +* Python slicing operators can address subsets of a list `L[0:3] == [e1,e2]`, `L[:-1] == en` +* Lists can be concatenated `L' = L + [ e3 ]` +* Lists can have entries removed `L' = L - [ e2, e1 ]`. This is different from Python's +`del` operation, which operates on the list in place. +* The membership predicate `in` returns true iff an element is a member of a List: `e2 in L` +* List comprehensions can create new lists: `L' = [ x for x in l where x < 5]` +* for a list `L`, `len(L)` returns the number of elements. + + +##### Sets + +Sets are an extension of the List notation, adding the restrictions that there can +be no duplicate entries in the set, and there is no defined order. + +* A set is an unordered collection of items surrounded by `{` and `}` braces. +* When declaring one, the python constructor `{}` is used. This is different from Python, which uses the function `set([list])`. Here the assumption +is that the difference between a set and a dictionary can be determined from the contents. +* The empty set `{}` has no elements. +* All the usual set concepts apply. +* The membership predicate is `in`. +* Set comprehension uses the Python list comprehension. +`S' = {s for s in S where len(s)==2}` +* for a set *s*, `len(s)` returns the number of elements. +* The `-` operator returns a new set excluding all items listed in the righthand set of the operator. + + + +##### Maps + +Maps resemble Python dictionaries; {"key":value, "key2",value2} + +* `keys(Map)` represents the set of keys in a map. +* `k in Map` holds iff `k in keys(Map)` +* The empty map is written `{:}` +* The `-` operator returns a new map which excludes the entry with the key specified. +* `len(Map)` returns the number of entries in the map. + +##### Strings + +Strings are lists of characters represented in double quotes. e.g. `"abc"` + + "abc" == ['a','b','c'] + +#### State Immutability + +All system state declarations are immutable. + +The suffix "'" (single quote) is used as the convention to indicate the state of the system after a operation: + + L' = L + ['d','e'] + + +#### Function Specifications + +A function is defined as a set of preconditions and a set of postconditions, +where the postconditions define the new state of the system and the return value from the function. + + +### Exceptions + +In classic specification languages, the preconditions define the predicates that MUST be +satisfied else some failure condition is raised. + +For Hadoop, we need to be able to specify what failure condition results if a specification is not +met (usually what exception is to be raised). + +The notation `raise ` is used to indicate that an exception is to be raised. + +It can be used in the if-then-else sequence to define an action if a precondition is not met. + +Example: + + if not exists(FS, Path) : raise IOException + +If implementations may raise any one of a set of exceptions, this is denoted by +providing a set of exceptions: + + if not exists(FS, Path) : raise {FileNotFoundException, IOException} + +If a set of exceptions is provided, the earlier elements +of the set are preferred to the later entries, on the basis that they aid diagnosis of problems. + +We also need to distinguish predicates that MUST be satisfied, along with those that SHOULD be met. +For this reason a function specification MAY include a section in the preconditions marked 'Should:' +All predicates declared in this section SHOULD be met, and if there is an entry in that section +which specifies a stricter outcome, it SHOULD BE preferred. Here is an example of a should-precondition: + +Should: + + if not exists(FS, Path) : raise FileNotFoundException + + +### Conditions + +There are further conditions used in precondition and postcondition declarations. + + +#### `supported(instance, method)` + + +This condition declares that a subclass implements the named method + -some subclasses of the verious FileSystem classes do not, and instead + raise `UnsupportedOperation` + +As an example, one precondition of `FSDataInputStream.seek` +is that the implementation must support `Seekable.seek` : + + supported(FDIS, Seekable.seek) else raise UnsupportedOperation diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/testing.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/testing.md new file mode 100644 index 00000000000..bc66e670468 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/testing.md @@ -0,0 +1,324 @@ + + +# Testing the Filesystem Contract + +## Running the tests + +A normal Hadoop test run will test those FileSystems that can be tested locally +via the local filesystem. This typically means `file://` and its underlying `LocalFileSystem`, and +`hdfs://` via the HDFS MiniCluster. + +Other filesystems are skipped unless there is a specific configuration to the +remote server providing the filesystem. + + +These filesystem bindings must be defined in an XML configuration file, usually +`hadoop-common-project/hadoop-common/src/test/resources/contract-test-options.xml`. +This file is excluded should not be checked in. + +### s3:// + +In `contract-test-options.xml`, the filesystem name must be defined in the property `fs.contract.test.fs.s3`. The standard configuration options to define the S3 authentication details must also be provided. + +Example: + + + + fs.contract.test.fs.s3 + s3://tests3hdfs/ + + + + fs.s3.awsAccessKeyId + DONOTPCOMMITTHISKEYTOSCM + + + + fs.s3.awsSecretAccessKey + DONOTEVERSHARETHISSECRETKEY! + + + +### s3n:// + + +In `contract-test-options.xml`, the filesystem name must be defined in the property `fs.contract.test.fs.s3n`. The standard configuration options to define the S3N authentication details muse also be provided. + +Example: + + + + + fs.contract.test.fs.s3n + s3n://tests3contract + + + + fs.s3n.awsAccessKeyId + DONOTPCOMMITTHISKEYTOSCM + + + + fs.s3n.awsSecretAccessKey + DONOTEVERSHARETHISSECRETKEY! + + +### ftp:// + + +In `contract-test-options.xml`, the filesystem name must be defined in +the property `fs.contract.test.fs.ftp`. The specific login options to +connect to the FTP Server must then be provided. + +A path to a test directory must also be provided in the option +`fs.contract.test.ftp.testdir`. This is the directory under which +operations take place. + +Example: + + + + + fs.contract.test.fs.ftp + ftp://server1/ + + + + fs.ftp.user.server1 + testuser + + + + fs.contract.test.ftp.testdir + /home/testuser/test + + + + fs.ftp.password.server1 + secret-login + + + + +### swift:// + +The OpenStack Swift login details must be defined in the file +`/hadoop-tools/hadoop-openstack/src/test/resources/contract-test-options.xml`. +The standard hadoop-common `contract-test-options.xml` resource file cannot be +used, as that file does not get included in `hadoop-common-test.jar`. + + +In `/hadoop-tools/hadoop-openstack/src/test/resources/contract-test-options.xml` +the Swift bucket name must be defined in the property `fs.contract.test.fs.swift`, +along with the login details for the specific Swift service provider in which the +bucket is posted. + + + + fs.contract.test.fs.swift + swift://swiftbucket.rackspace/ + + + + fs.swift.service.rackspace.auth.url + https://auth.api.rackspacecloud.com/v2.0/tokens + Rackspace US (multiregion) + + + + fs.swift.service.rackspace.username + this-is-your-username + + + + fs.swift.service.rackspace.region + DFW + + + + fs.swift.service.rackspace.apikey + ab0bceyoursecretapikeyffef + + + + +1. Often the different public cloud Swift infrastructures exhibit different behaviors +(authentication and throttling in particular). We recommand that testers create +accounts on as many of these providers as possible and test against each of them. +1. They can be slow, especially remotely. Remote links are also the most likely +to make eventual-consistency behaviors visible, which is a mixed benefit. + +## Testing a new filesystem + +The core of adding a new FileSystem to the contract tests is adding a +new contract class, then creating a new non-abstract test class for every test +suite that you wish to test. + +1. Do not try and add these tests into Hadoop itself. They won't be added to +the soutce tree. The tests must live with your own filesystem source. +1. Create a package in your own test source tree (usually) under `contract`, +for the files and tests. +1. Subclass `AbstractFSContract` for your own contract implementation. +1. For every test suite you plan to support create a non-abstract subclass, + with the name starting with `Test` and the name of the filesystem. + Example: `TestHDFSRenameContract`. +1. These non-abstract classes must implement the abstract method + `createContract()`. +1. Identify and document any filesystem bindings that must be defined in a + `src/test/resources/contract-test-options.xml` file of the specific project. +1. Run the tests until they work. + + +As an example, here is the implementation of the test of the `create()` tests for the local filesystem. + + package org.apache.hadoop.fs.contract.localfs; + + import org.apache.hadoop.conf.Configuration; + import org.apache.hadoop.fs.contract.AbstractCreateContractTest; + import org.apache.hadoop.fs.contract.AbstractFSContract; + + public class TestLocalCreateContract extends AbstractCreateContractTest { + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new LocalFSContract(conf); + } + } + +The standard implementation technique for subclasses of `AbstractFSContract` is to be driven entirely by a Hadoop XML configuration file stored in the test resource tree. The best practise is to store it under `/contract` with the name of the FileSystem, such as `contract/localfs.xml`. Having the XML file define all FileSystem options makes the listing of FileSystem behaviors immediately visible. + +The `LocalFSContract` is a special case of this, as it must adjust its case sensitivity policy based on the OS on which it is running: for both Windows and OS/X, the filesystem is case insensitive, so the `ContractOptions.IS_CASE_SENSITIVE` option must be set to false. Furthermore, the Windows filesystem does not support Unix file and directory permissions, so the relevant flag must also be set. This is done *after* loading the XML contract file from the resource tree, simply by updating the now-loaded configuration options: + + getConf().setBoolean(getConfKey(ContractOptions.SUPPORTS_UNIX_PERMISSIONS), false); + + + +### Handling test failures + +If your new `FileSystem` test cases fails one of the contract tests, what you can you do? + +It depends on the cause of the problem + +1. Case: custom `FileSystem` subclass class doesn't correctly implement specification. Fix. +1. Case: Underlying filesystem doesn't behave in a way that matches Hadoop's expectations. Ideally, fix. Or try to make your `FileSystem` subclass hide the differences, e.g. by translating exceptions. +1. Case: fundamental architectural differences between your filesystem and Hadoop. Example: different concurrency and consistency model. Recommendation: document and make clear that the filesystem is not compatible with HDFS. +1. Case: test does not match the specification. Fix: patch test, submit the patch to Hadoop. +1. Case: specification incorrect. The underlying specification is (with a few exceptions) HDFS. If the specification does not match HDFS, HDFS should normally be assumed to be the real definition of what a FileSystem should do. If there's a mismatch, please raise it on the `hdfs-dev` mailing list. Note that while FileSystem tests live in the core Hadoop codebase, it is the HDFS team who owns the FileSystem specification and the tests that accompany it. + +If a test needs to be skipped because a feature is not supported, look for a existing configuration option in the `ContractOptions` class. If there is no method, the short term fix is to override the method and use the `ContractTestUtils.skip()` message to log the fact that a test is skipped. Using this method prints the message to the logs, then tells the test runner that the test was skipped. This highlights the problem. + +A recommended strategy is to call the superclass, catch the exception, and verify that the exception class and part of the error string matches that raised by the current implementation. It should also `fail()` if superclass actually succeeded -that is it failed the way that the implemention does not currently do. This will ensure that the test path is still executed, any other failure of the test -possibly a regression- is picked up. And, if the feature does become implemented, that the change is picked up. + +A long-term solution is to enhance the base test to add a new optional feature key. This will require collaboration with the developers on the `hdfs-dev` mailing list. + + + +### 'Lax vs Strict' exceptions + +The contract tests include the notion of strict vs lax exceptions. *Strict* exception reporting means: reports failures using specific subclasses of `IOException`, such as `FileNotFoundException`, `EOFException` and so on. *Lax* reporting means throws `IOException`. + +While FileSystems SHOULD raise stricter exceptions, there may be reasons why they cannot. Raising lax exceptions is still allowed, it merely hampers diagnostics of failures in user applications. To declare that a FileSystem does not support the stricter exceptions, set the option `fs.contract.supports-strict-exceptions` to false. + +### Supporting FileSystems with login and authentication parameters + +Tests against remote FileSystems will require the URL to the FileSystem to be specified; +tests against remote FileSystems that require login details require usernames/IDs and passwords. + +All these details MUST be required to be placed in the file `src/test/resources/contract-test-options.xml`, and your SCM tools configured to never commit this file to subversion, git or +equivalent. Furthermore, the build MUST be configured to never bundle this file in any `-test` artifacts generated. The Hadoop build does this, excluding `src/test/**/*.xml` from the JAR files. + +The `AbstractFSContract` class automatically loads this resource file if present; specific keys for specific test cases can be added. + +As an example, here are what S3N test keys look like: + + + + fs.contract.test.fs.s3n + s3n://tests3contract + + + + fs.s3n.awsAccessKeyId + DONOTPCOMMITTHISKEYTOSCM + + + + fs.s3n.awsSecretAccessKey + DONOTEVERSHARETHISSECRETKEY! + + + +The `AbstractBondedFSContract` automatically skips a test suite if the FileSystem URL is not defined in the property `fs.contract.test.fs.%s`, where `%s` matches the schema name of the FileSystem. + + + +### Important: passing the tests does not guarantee compatibility + +Passing all the FileSystem contract tests does not mean that a filesystem can be described as "compatible with HDFS". The tests try to look at the isolated functionality of each operation, and focus on the preconditions and postconditions of each action. Core areas not covered are concurrency and aspects of failure across a distributed system. + +* Consistency: are all changes immediately visible? +* Atomicity: are operations which HDFS guarantees to be atomic equally so on the new filesystem. +* Idempotency: if the filesystem implements any retry policy, is idempotent even while other clients manipulate the filesystem? +* Scalability: does it support files as large as HDFS, or as many in a single directory? +* Durability: do files actually last -and how long for? + +Proof that this is is true is the fact that the Amazon S3 and OpenStack Swift object stores are eventually consistent object stores with non-atomic rename and delete operations. Single threaded test cases are unlikely to see some of the concurrency issues, while consistency is very often only visible in tests that span a datacenter. + +There are also some specific aspects of the use of the FileSystem API: + +* Compatibility with the `hadoop -fs` CLI. +* Whether the blocksize policy produces file splits that are suitable for analytics workss. (as an example, a blocksize of 1 matches the specification, but as it tells MapReduce jobs to work a byte at a time, unusable). + +Tests that verify these behaviors are of course welcome. + + + +## Adding a new test suite + +1. New tests should be split up with a test class per operation, as is done for `seek()`, `rename()`, `create()`, and so on. This is to match up the way that the FileSystem contract specification is split up by operation. It also makes it easier for FileSystem implementors to work on one test suite at a time. +2. Subclass `AbstractFSContractTestBase` with a new abstract test suite class. Again, use `Abstract` in the title. +3. Look at `org.apache.hadoop.fs.contract.ContractTestUtils` for utility classes to aid testing, with lots of filesystem-centric assertions. Use these to make assertions about the filesystem state, and to incude diagnostics information such as directory listings and dumps of mismatched files when an assertion actually fails. +4. Write tests for the local, raw local and HDFS filesystems -if one of these fails the tests then there is a sign of a problem -though be aware that they do have differnces +5. Test on the object stores once the core filesystems are passing the tests. +4. Try and log failures with as much detail as you can -the people debugging the failures will appreciate it. + + +### Root manipulation tests + +Some tests work directly against the root filesystem, attempting to do things like rename "/" and similar actions. The root directory is "special", and it's important to test this, especially on non-POSIX filesystems such as object stores. These tests are potentially very destructive to native filesystems, so use care. + +1. Add the tests under `AbstractRootDirectoryContractTest` or create a new test with (a) `Root` in the title and (b) a check in the setup method to skip the test if root tests are disabled: + + skipIfUnsupported(TEST_ROOT_TESTS_ENABLED); + +1. Don't provide an implementation of this test suite to run against the local FS. + +### Scalability tests + +Tests designed to generate scalable load -and that includes a large number of small files, as well as fewer larger files, should be designed to be configurable, so that users of the test +suite can configure the number and size of files. + +Be aware that on object stores, the directory rename operation is usually `O(files)*O(data)` while the delete operation is `O(files)`. The latter means even any directory cleanup operations may take time and can potentially timeout. It is important to design tests that work against remote filesystems with possible delays in all operations. + +## Extending the specification + +The specification is incomplete. It doesn't have complete coverage of the FileSystem classes, and there may be bits of the existing specified classes that are not covered. + +1. Look at the implementations of a class/interface/method to see what they do, especially HDFS and local. These are the documentation of what is done today. +2. Look at the POSIX API specification. +3. Search through the HDFS JIRAs for discussions on FileSystem topics, and try to understand what was meant to happen, as well as what does happen. +4. Use an IDE to find out how methods are used in Hadoop, HBase and other parts of the stack. Although this assumes that these are representative Hadoop applications, it will at least show how applications *expect* a FileSystem to behave. +5. Look in the java.io source to see how the bunded FileSystem classes are expected to behave -and read their javadocs carefully. +5. If something is unclear -as on the hdfs-dev list. +6. Don't be afraid to write tests to act as experiments and clarify what actually happens. Use the HDFS behaviours as the normative guide. diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java index 48dc9ede877..73a92a177ce 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java @@ -227,7 +227,7 @@ public void testCreateFileAndMkdirs() throws IOException { try { fileSys.mkdirs(bad_dir); fail("Failed to detect existing file in path"); - } catch (FileAlreadyExistsException e) { + } catch (ParentNotDirectoryException e) { // Expected } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractBondedFSContract.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractBondedFSContract.java new file mode 100644 index 00000000000..24756393587 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractBondedFSContract.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; + +/** + * This is a filesystem contract for any class that bonds to a filesystem + * through the configuration. + * + * It looks for a definition of the test filesystem with the key + * derived from "fs.contract.test.fs.%s" -if found the value + * is converted to a URI and used to create a filesystem. If not -the + * tests are not enabled + */ +public abstract class AbstractBondedFSContract extends AbstractFSContract { + + private static final Log LOG = + LogFactory.getLog(AbstractBondedFSContract.class); + + /** + * Pattern for the option for test filesystems from schema + */ + public static final String FSNAME_OPTION = "test.fs.%s"; + + /** + * Constructor: loads the authentication keys if found + + * @param conf configuration to work with + */ + protected AbstractBondedFSContract(Configuration conf) { + super(conf); + } + + private String fsName; + private URI fsURI; + private FileSystem filesystem; + + @Override + public void init() throws IOException { + super.init(); + //this test is only enabled if the test FS is present + fsName = loadFilesystemName(getScheme()); + setEnabled(!fsName.isEmpty()); + if (isEnabled()) { + try { + fsURI = new URI(fsName); + filesystem = FileSystem.get(fsURI, getConf()); + } catch (URISyntaxException e) { + throw new IOException("Invalid URI " + fsName); + } catch (IllegalArgumentException e) { + throw new IOException("Invalid URI " + fsName, e); + } + } else { + LOG.info("skipping tests as FS name is not defined in " + + getFilesystemConfKey()); + } + } + + /** + * Load the name of a test filesystem. + * @param schema schema to look up + * @return the filesystem name -or "" if none was defined + */ + public String loadFilesystemName(String schema) { + return getOption(String.format(FSNAME_OPTION, schema), ""); + } + + /** + * Get the conf key for a filesystem + */ + protected String getFilesystemConfKey() { + return getConfKey(String.format(FSNAME_OPTION, getScheme())); + } + + @Override + public FileSystem getTestFileSystem() throws IOException { + return filesystem; + } + + @Override + public Path getTestPath() { + Path path = new Path("/test"); + return path; + } + + @Override + public String toString() { + return getScheme() +" Contract against " + fsName; + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractAppendTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractAppendTest.java new file mode 100644 index 00000000000..96287057a05 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractAppendTest.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract; + +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.Path; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.hadoop.fs.contract.ContractTestUtils.cleanup; +import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile; +import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; +import static org.apache.hadoop.fs.contract.ContractTestUtils.touch; + +/** + * Test concat -if supported + */ +public abstract class AbstractContractAppendTest extends AbstractFSContractTestBase { + private static final Logger LOG = + LoggerFactory.getLogger(AbstractContractAppendTest.class); + + private Path testPath; + private Path target; + + @Override + public void setup() throws Exception { + super.setup(); + skipIfUnsupported(SUPPORTS_APPEND); + + //delete the test directory + testPath = path("test"); + target = new Path(testPath, "target"); + } + + @Test + public void testAppendToEmptyFile() throws Throwable { + touch(getFileSystem(), target); + byte[] dataset = dataset(256, 'a', 'z'); + FSDataOutputStream outputStream = getFileSystem().append(target); + try { + outputStream.write(dataset); + } finally { + outputStream.close(); + } + byte[] bytes = ContractTestUtils.readDataset(getFileSystem(), target, + dataset.length); + ContractTestUtils.compareByteArrays(dataset, bytes, dataset.length); + } + + @Test + public void testAppendNonexistentFile() throws Throwable { + try { + FSDataOutputStream out = getFileSystem().append(target); + //got here: trouble + out.close(); + fail("expected a failure"); + } catch (Exception e) { + //expected + handleExpectedException(e); + } + } + + @Test + public void testAppendToExistingFile() throws Throwable { + byte[] original = dataset(8192, 'A', 'Z'); + byte[] appended = dataset(8192, '0', '9'); + createFile(getFileSystem(), target, false, original); + FSDataOutputStream outputStream = getFileSystem().append(target); + outputStream.write(appended); + outputStream.close(); + byte[] bytes = ContractTestUtils.readDataset(getFileSystem(), target, + original.length + appended.length); + ContractTestUtils.validateFileContent(bytes, + new byte[] [] { original, appended }); + } + + @Test + public void testAppendMissingTarget() throws Throwable { + try { + FSDataOutputStream out = getFileSystem().append(target); + //got here: trouble + out.close(); + fail("expected a failure"); + } catch (Exception e) { + //expected + handleExpectedException(e); + } + } + + @Test + public void testRenameFileBeingAppended() throws Throwable { + touch(getFileSystem(), target); + assertPathExists("original file does not exist", target); + byte[] dataset = dataset(256, 'a', 'z'); + FSDataOutputStream outputStream = getFileSystem().append(target); + outputStream.write(dataset); + Path renamed = new Path(testPath, "renamed"); + outputStream.close(); + String listing = ls(testPath); + + //expected: the stream goes to the file that was being renamed, not + //the original path + assertPathExists("renamed destination file does not exist", renamed); + + assertPathDoesNotExist("Source file found after rename during append:\n" + + listing, target); + byte[] bytes = ContractTestUtils.readDataset(getFileSystem(), renamed, + dataset.length); + ContractTestUtils.compareByteArrays(dataset, bytes, dataset.length); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractConcatTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractConcatTest.java new file mode 100644 index 00000000000..69e902b1b0e --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractConcatTest.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract; + +import org.apache.hadoop.fs.Path; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.hadoop.fs.contract.ContractTestUtils.assertFileHasLength; +import static org.apache.hadoop.fs.contract.ContractTestUtils.cleanup; +import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile; +import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; +import static org.apache.hadoop.fs.contract.ContractTestUtils.touch; + +/** + * Test concat -if supported + */ +public abstract class AbstractContractConcatTest extends AbstractFSContractTestBase { + private static final Logger LOG = + LoggerFactory.getLogger(AbstractContractConcatTest.class); + + private Path testPath; + private Path srcFile; + private Path zeroByteFile; + private Path target; + + @Override + public void setup() throws Exception { + super.setup(); + skipIfUnsupported(SUPPORTS_CONCAT); + + //delete the test directory + testPath = path("test"); + srcFile = new Path(testPath, "small.txt"); + zeroByteFile = new Path(testPath, "zero.txt"); + target = new Path(testPath, "target"); + + byte[] block = dataset(TEST_FILE_LEN, 0, 255); + createFile(getFileSystem(), srcFile, false, block); + touch(getFileSystem(), zeroByteFile); + } + + @Test + public void testConcatEmptyFiles() throws Throwable { + touch(getFileSystem(), target); + try { + getFileSystem().concat(target, new Path[0]); + fail("expected a failure"); + } catch (Exception e) { + //expected + handleExpectedException(e); + } + } + + @Test + public void testConcatMissingTarget() throws Throwable { + try { + getFileSystem().concat(target, + new Path[] { zeroByteFile}); + fail("expected a failure"); + } catch (Exception e) { + //expected + handleExpectedException(e); + } + } + + @Test + public void testConcatFileOnFile() throws Throwable { + byte[] block = dataset(TEST_FILE_LEN, 0, 255); + createFile(getFileSystem(), target, false, block); + getFileSystem().concat(target, + new Path[] {srcFile}); + assertFileHasLength(getFileSystem(), target, TEST_FILE_LEN *2); + ContractTestUtils.validateFileContent( + ContractTestUtils.readDataset(getFileSystem(), + target, TEST_FILE_LEN * 2), + new byte[][]{block, block}); + } + + @Test + public void testConcatOnSelf() throws Throwable { + byte[] block = dataset(TEST_FILE_LEN, 0, 255); + createFile(getFileSystem(), target, false, block); + try { + getFileSystem().concat(target, + new Path[]{target}); + } catch (Exception e) { + //expected + handleExpectedException(e); + } + } + + + +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractCreateTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractCreateTest.java new file mode 100644 index 00000000000..f42ab781873 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractCreateTest.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract; + +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileAlreadyExistsException; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IOUtils; +import org.junit.Test; +import org.junit.internal.AssumptionViolatedException; + +import java.io.FileNotFoundException; +import java.io.IOException; + +import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; +import static org.apache.hadoop.fs.contract.ContractTestUtils.skip; +import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset; +import static org.apache.hadoop.fs.contract.ContractTestUtils.writeTextFile; + +/** + * Test creating files, overwrite options &c + */ +public abstract class AbstractContractCreateTest extends + AbstractFSContractTestBase { + + @Test + public void testCreateNewFile() throws Throwable { + describe("Foundational 'create a file' test"); + Path path = path("testCreateNewFile"); + byte[] data = dataset(256, 'a', 'z'); + writeDataset(getFileSystem(), path, data, data.length, 1024 * 1024, false); + ContractTestUtils.verifyFileContents(getFileSystem(), path, data); + } + + @Test + public void testCreateFileOverExistingFileNoOverwrite() throws Throwable { + describe("Verify overwriting an existing file fails"); + Path path = path("testCreateFileOverExistingFileNoOverwrite"); + byte[] data = dataset(256, 'a', 'z'); + writeDataset(getFileSystem(), path, data, data.length, 1024, false); + byte[] data2 = dataset(10 * 1024, 'A', 'Z'); + try { + writeDataset(getFileSystem(), path, data2, data2.length, 1024, false); + fail("writing without overwrite unexpectedly succeeded"); + } catch (FileAlreadyExistsException expected) { + //expected + handleExpectedException(expected); + } catch (IOException relaxed) { + handleRelaxedException("Creating a file over a file with overwrite==false", + "FileAlreadyExistsException", + relaxed); + } + } + + /** + * This test catches some eventual consistency problems that blobstores exhibit, + * as we are implicitly verifying that updates are consistent. This + * is why different file lengths and datasets are used + * @throws Throwable + */ + @Test + public void testOverwriteExistingFile() throws Throwable { + describe("Overwrite an existing file and verify the new data is there"); + Path path = path("testOverwriteExistingFile"); + byte[] data = dataset(256, 'a', 'z'); + writeDataset(getFileSystem(), path, data, data.length, 1024, false); + ContractTestUtils.verifyFileContents(getFileSystem(), path, data); + byte[] data2 = dataset(10 * 1024, 'A', 'Z'); + writeDataset(getFileSystem(), path, data2, data2.length, 1024, true); + ContractTestUtils.verifyFileContents(getFileSystem(), path, data2); + } + + @Test + public void testOverwriteEmptyDirectory() throws Throwable { + describe("verify trying to create a file over an empty dir fails"); + Path path = path("testOverwriteEmptyDirectory"); + mkdirs(path); + assertIsDirectory(path); + byte[] data = dataset(256, 'a', 'z'); + try { + writeDataset(getFileSystem(), path, data, data.length, 1024, true); + assertIsDirectory(path); + fail("write of file over empty dir succeeded"); + } catch (FileAlreadyExistsException expected) { + //expected + handleExpectedException(expected); + } catch (FileNotFoundException e) { + handleRelaxedException("overwriting a dir with a file ", + "FileAlreadyExistsException", + e); + } catch (IOException e) { + handleRelaxedException("overwriting a dir with a file ", + "FileAlreadyExistsException", + e); + } + assertIsDirectory(path); + } + + @Test + public void testOverwriteNonEmptyDirectory() throws Throwable { + describe("verify trying to create a file over a non-empty dir fails"); + Path path = path("testOverwriteNonEmptyDirectory"); + mkdirs(path); + try { + assertIsDirectory(path); + } catch (AssertionError failure) { + if (isSupported(IS_BLOBSTORE)) { + // file/directory hack surfaces here + throw new AssumptionViolatedException(failure.toString()).initCause(failure); + } + // else: rethrow + throw failure; + } + Path child = new Path(path, "child"); + writeTextFile(getFileSystem(), child, "child file", true); + byte[] data = dataset(256, 'a', 'z'); + try { + writeDataset(getFileSystem(), path, data, data.length, 1024, + true); + FileStatus status = getFileSystem().getFileStatus(path); + + boolean isDir = status.isDirectory(); + if (!isDir && isSupported(IS_BLOBSTORE)) { + // object store: downgrade to a skip so that the failure is visible + // in test results + skip("Object store allows a file to overwrite a directory"); + } + fail("write of file over dir succeeded"); + } catch (FileAlreadyExistsException expected) { + //expected + handleExpectedException(expected); + } catch (FileNotFoundException e) { + handleRelaxedException("overwriting a dir with a file ", + "FileAlreadyExistsException", + e); + } catch (IOException e) { + handleRelaxedException("overwriting a dir with a file ", + "FileAlreadyExistsException", + e); + } + assertIsDirectory(path); + assertIsFile(child); + } + + @Test + public void testCreatedFileIsImmediatelyVisible() throws Throwable { + describe("verify that a newly created file exists as soon as open returns"); + Path path = path("testCreatedFileIsImmediatelyVisible"); + FSDataOutputStream out = null; + try { + out = getFileSystem().create(path, + false, + 4096, + (short) 1, + 1024); + if (!getFileSystem().exists(path)) { + + if (isSupported(IS_BLOBSTORE)) { + // object store: downgrade to a skip so that the failure is visible + // in test results + skip("Filesystem is an object store and newly created files are not immediately visible"); + } + assertPathExists("expected path to be visible before anything written", + path); + } + } finally { + IOUtils.closeStream(out); + } + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractDeleteTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractDeleteTest.java new file mode 100644 index 00000000000..c90efd19386 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractDeleteTest.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract; + +import org.apache.hadoop.fs.Path; +import org.junit.Test; + +import java.io.IOException; + +/** + * Test creating files, overwrite options &c + */ +public abstract class AbstractContractDeleteTest extends + AbstractFSContractTestBase { + + @Test + public void testDeleteEmptyDirNonRecursive() throws Throwable { + Path path = path("testDeleteEmptyDirNonRecursive"); + mkdirs(path); + assertDeleted(path, false); + } + + @Test + public void testDeleteEmptyDirRecursive() throws Throwable { + Path path = path("testDeleteEmptyDirRecursive"); + mkdirs(path); + assertDeleted(path, true); + } + + @Test + public void testDeleteNonexistentPathRecursive() throws Throwable { + Path path = path("testDeleteNonexistentPathRecursive"); + ContractTestUtils.assertPathDoesNotExist(getFileSystem(), "leftover", path); + ContractTestUtils.rejectRootOperation(path); + assertFalse("Returned true attempting to delete" + + " a nonexistent path " + path, + getFileSystem().delete(path, false)); + } + + + @Test + public void testDeleteNonexistentPathNonRecursive() throws Throwable { + Path path = path("testDeleteNonexistentPathNonRecursive"); + ContractTestUtils.assertPathDoesNotExist(getFileSystem(), "leftover", path); + ContractTestUtils.rejectRootOperation(path); + assertFalse("Returned true attempting to recursively delete" + + " a nonexistent path " + path, + getFileSystem().delete(path, false)); + } + + @Test + public void testDeleteNonEmptyDirNonRecursive() throws Throwable { + Path path = path("testDeleteNonEmptyDirNonRecursive"); + mkdirs(path); + Path file = new Path(path, "childfile"); + ContractTestUtils.writeTextFile(getFileSystem(), file, "goodbye, world", + true); + try { + ContractTestUtils.rejectRootOperation(path); + boolean deleted = getFileSystem().delete(path, false); + fail("non recursive delete should have raised an exception," + + " but completed with exit code " + deleted); + } catch (IOException expected) { + //expected + handleExpectedException(expected); + } + ContractTestUtils.assertIsDirectory(getFileSystem(), path); + } + + @Test + public void testDeleteNonEmptyDirRecursive() throws Throwable { + Path path = path("testDeleteNonEmptyDirNonRecursive"); + mkdirs(path); + Path file = new Path(path, "childfile"); + ContractTestUtils.writeTextFile(getFileSystem(), file, "goodbye, world", + true); + assertDeleted(path, true); + ContractTestUtils.assertPathDoesNotExist(getFileSystem(), "not deleted", file); + } + +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMkdirTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMkdirTest.java new file mode 100644 index 00000000000..dad3b7f2c46 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMkdirTest.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract; + +import org.apache.hadoop.fs.FileAlreadyExistsException; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.ParentNotDirectoryException; +import org.apache.hadoop.fs.Path; +import org.junit.Test; + +import java.io.IOException; + +import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile; +import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; + +/** + * Test directory operations + */ +public abstract class AbstractContractMkdirTest extends AbstractFSContractTestBase { + + @Test + public void testMkDirRmDir() throws Throwable { + FileSystem fs = getFileSystem(); + + Path dir = path("testMkDirRmDir"); + assertPathDoesNotExist("directory already exists", dir); + fs.mkdirs(dir); + assertPathExists("mkdir failed", dir); + assertDeleted(dir, false); + } + + @Test + public void testMkDirRmRfDir() throws Throwable { + describe("create a directory then recursive delete it"); + FileSystem fs = getFileSystem(); + Path dir = path("testMkDirRmRfDir"); + assertPathDoesNotExist("directory already exists", dir); + fs.mkdirs(dir); + assertPathExists("mkdir failed", dir); + assertDeleted(dir, true); + } + + @Test + public void testNoMkdirOverFile() throws Throwable { + describe("try to mkdir over a file"); + FileSystem fs = getFileSystem(); + Path path = path("testNoMkdirOverFile"); + byte[] dataset = dataset(1024, ' ', 'z'); + createFile(getFileSystem(), path, false, dataset); + try { + boolean made = fs.mkdirs(path); + fail("mkdirs did not fail over a file but returned " + made + + "; " + ls(path)); + } catch (ParentNotDirectoryException e) { + //parent is a directory + handleExpectedException(e); + } catch (FileAlreadyExistsException e) { + //also allowed as an exception (HDFS) + handleExpectedException(e);; + } catch (IOException e) { + //here the FS says "no create" + handleRelaxedException("mkdirs", "FileAlreadyExistsException", e); + } + assertIsFile(path); + byte[] bytes = ContractTestUtils.readDataset(getFileSystem(), path, + dataset.length); + ContractTestUtils.compareByteArrays(dataset, bytes, dataset.length); + assertPathExists("mkdir failed", path); + assertDeleted(path, true); + } + + @Test + public void testMkdirOverParentFile() throws Throwable { + describe("try to mkdir where a parent is a file"); + FileSystem fs = getFileSystem(); + Path path = path("testMkdirOverParentFile"); + byte[] dataset = dataset(1024, ' ', 'z'); + createFile(getFileSystem(), path, false, dataset); + Path child = new Path(path,"child-to-mkdir"); + try { + boolean made = fs.mkdirs(child); + fail("mkdirs did not fail over a file but returned " + made + + "; " + ls(path)); + } catch (ParentNotDirectoryException e) { + //parent is a directory + handleExpectedException(e); + } catch (FileAlreadyExistsException e) { + handleExpectedException(e); + } catch (IOException e) { + handleRelaxedException("mkdirs", "ParentNotDirectoryException", e); + } + assertIsFile(path); + byte[] bytes = ContractTestUtils.readDataset(getFileSystem(), path, + dataset.length); + ContractTestUtils.compareByteArrays(dataset, bytes, dataset.length); + assertPathExists("mkdir failed", path); + assertDeleted(path, true); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java new file mode 100644 index 00000000000..65ebfb19466 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractOpenTest.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IOUtils; +import org.junit.Test; + +import java.io.FileNotFoundException; +import java.io.IOException; + +import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile; +import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; +import static org.apache.hadoop.fs.contract.ContractTestUtils.touch; + +/** + * Test Seek operations + */ +public abstract class AbstractContractOpenTest extends AbstractFSContractTestBase { + + private FSDataInputStream instream; + + @Override + protected Configuration createConfiguration() { + Configuration conf = super.createConfiguration(); + conf.setInt(CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY, 4096); + return conf; + } + + @Override + public void teardown() throws Exception { + IOUtils.closeStream(instream); + instream = null; + super.teardown(); + } + + @Test + public void testOpenReadZeroByteFile() throws Throwable { + describe("create & read a 0 byte file"); + Path path = path("zero.txt"); + touch(getFileSystem(), path); + instream = getFileSystem().open(path); + assertEquals(0, instream.getPos()); + //expect initial read to fail + int result = instream.read(); + assertMinusOne("initial byte read", result); + } + + @Test + public void testOpenReadDir() throws Throwable { + describe("create & read a directory"); + Path path = path("zero.dir"); + mkdirs(path); + try { + instream = getFileSystem().open(path); + //at this point we've opened a directory + fail("A directory has been opened for reading"); + } catch (FileNotFoundException e) { + handleExpectedException(e); + } catch (IOException e) { + handleRelaxedException("opening a directory for reading", + "FileNotFoundException", + e); + } + } + + @Test + public void testOpenReadDirWithChild() throws Throwable { + describe("create & read a directory which has a child"); + Path path = path("zero.dir"); + mkdirs(path); + Path path2 = new Path(path, "child"); + mkdirs(path2); + + try { + instream = getFileSystem().open(path); + //at this point we've opened a directory + fail("A directory has been opened for reading"); + } catch (FileNotFoundException e) { + handleExpectedException(e); + } catch (IOException e) { + handleRelaxedException("opening a directory for reading", + "FileNotFoundException", + e); + } + } + + @Test + public void testOpenFileTwice() throws Throwable { + describe("verify that two opened file streams are independent"); + Path path = path("testopenfiletwice.txt"); + byte[] block = dataset(TEST_FILE_LEN, 0, 255); + //this file now has a simple rule: offset => value + createFile(getFileSystem(), path, false, block); + //open first + FSDataInputStream instream1 = getFileSystem().open(path); + int c = instream1.read(); + assertEquals(0,c); + FSDataInputStream instream2 = null; + try { + instream2 = getFileSystem().open(path); + assertEquals("first read of instream 2", 0, instream2.read()); + assertEquals("second read of instream 1", 1, instream1.read()); + instream1.close(); + assertEquals("second read of instream 2", 1, instream2.read()); + //close instream1 again + instream1.close(); + } finally { + IOUtils.closeStream(instream1); + IOUtils.closeStream(instream2); + } + } + + @Test + public void testSequentialRead() throws Throwable { + describe("verify that sequential read() operations return values"); + Path path = path("testsequentialread.txt"); + int len = 4; + int base = 0x40; // 64 + byte[] block = dataset(len, base, base + len); + //this file now has a simple rule: offset => (value | 0x40) + createFile(getFileSystem(), path, false, block); + //open first + instream = getFileSystem().open(path); + assertEquals(base, instream.read()); + assertEquals(base + 1, instream.read()); + assertEquals(base + 2, instream.read()); + assertEquals(base + 3, instream.read()); + // and now, failures + assertEquals(-1, instream.read()); + assertEquals(-1, instream.read()); + instream.close(); + } + + +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractRenameTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractRenameTest.java new file mode 100644 index 00000000000..32f27a713fb --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractRenameTest.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract; + +import org.apache.hadoop.fs.FileAlreadyExistsException; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.junit.Test; + +import java.io.FileNotFoundException; +import java.io.IOException; + +import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; +import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset; + +/** + * Test creating files, overwrite options &c + */ +public abstract class AbstractContractRenameTest extends + AbstractFSContractTestBase { + + @Test + public void testRenameNewFileSameDir() throws Throwable { + describe("rename a file into a new file in the same directory"); + Path renameSrc = path("rename_src"); + Path renameTarget = path("rename_dest"); + byte[] data = dataset(256, 'a', 'z'); + writeDataset(getFileSystem(), renameSrc, + data, data.length, 1024 * 1024, false); + boolean rename = rename(renameSrc, renameTarget); + assertTrue("rename("+renameSrc+", "+ renameTarget+") returned false", + rename); + ContractTestUtils.assertListStatusFinds(getFileSystem(), + renameTarget.getParent(), renameTarget); + ContractTestUtils.verifyFileContents(getFileSystem(), renameTarget, data); + } + + @Test + public void testRenameNonexistentFile() throws Throwable { + describe("rename a file into a new file in the same directory"); + Path missing = path("testRenameNonexistentFileSrc"); + Path target = path("testRenameNonexistentFileDest"); + boolean renameReturnsFalseOnFailure = + isSupported(ContractOptions.RENAME_RETURNS_FALSE_IF_SOURCE_MISSING); + mkdirs(missing.getParent()); + try { + boolean renamed = rename(missing, target); + //expected an exception + if (!renameReturnsFalseOnFailure) { + String destDirLS = generateAndLogErrorListing(missing, target); + fail("expected rename(" + missing + ", " + target + " ) to fail," + + " got a result of " + renamed + + " and a destination directory of " + destDirLS); + } else { + // at least one FS only returns false here, if that is the case + // warn but continue + getLog().warn("Rename returned {} renaming a nonexistent file", renamed); + assertFalse("Renaming a missing file returned true", renamed); + } + } catch (FileNotFoundException e) { + if (renameReturnsFalseOnFailure) { + ContractTestUtils.fail( + "Renaming a missing file unexpectedly threw an exception", e); + } + handleExpectedException(e); + } catch (IOException e) { + handleRelaxedException("rename nonexistent file", + "FileNotFoundException", + e); + } + assertPathDoesNotExist("rename nonexistent file created a destination file", target); + } + + /** + * Rename test -handles filesystems that will overwrite the destination + * as well as those that do not (i.e. HDFS). + * @throws Throwable + */ + @Test + public void testRenameFileOverExistingFile() throws Throwable { + describe("Verify renaming a file onto an existing file matches expectations"); + Path srcFile = path("source-256.txt"); + byte[] srcData = dataset(256, 'a', 'z'); + writeDataset(getFileSystem(), srcFile, srcData, srcData.length, 1024, false); + Path destFile = path("dest-512.txt"); + byte[] destData = dataset(512, 'A', 'Z'); + writeDataset(getFileSystem(), destFile, destData, destData.length, 1024, false); + assertIsFile(destFile); + boolean renameOverwritesDest = isSupported(RENAME_OVERWRITES_DEST); + boolean renameReturnsFalseOnRenameDestExists = + !isSupported(RENAME_RETURNS_FALSE_IF_DEST_EXISTS); + boolean destUnchanged = true; + try { + boolean renamed = rename(srcFile, destFile); + + if (renameOverwritesDest) { + // the filesystem supports rename(file, file2) by overwriting file2 + + assertTrue("Rename returned false", renamed); + destUnchanged = false; + } else { + // rename is rejected by returning 'false' or throwing an exception + if (renamed && !renameReturnsFalseOnRenameDestExists) { + //expected an exception + String destDirLS = generateAndLogErrorListing(srcFile, destFile); + getLog().error("dest dir {}", destDirLS); + fail("expected rename(" + srcFile + ", " + destFile + " ) to fail," + + " but got success and destination of " + destDirLS); + } + } + } catch (FileAlreadyExistsException e) { + handleExpectedException(e); + } + // verify that the destination file is as expected based on the expected + // outcome + ContractTestUtils.verifyFileContents(getFileSystem(), destFile, + destUnchanged? destData: srcData); + } + + @Test + public void testRenameDirIntoExistingDir() throws Throwable { + describe("Verify renaming a dir into an existing dir puts it underneath" + +" and leaves existing files alone"); + FileSystem fs = getFileSystem(); + String sourceSubdir = "source"; + Path srcDir = path(sourceSubdir); + Path srcFilePath = new Path(srcDir, "source-256.txt"); + byte[] srcDataset = dataset(256, 'a', 'z'); + writeDataset(fs, srcFilePath, srcDataset, srcDataset.length, 1024, false); + Path destDir = path("dest"); + + Path destFilePath = new Path(destDir, "dest-512.txt"); + byte[] destDateset = dataset(512, 'A', 'Z'); + writeDataset(fs, destFilePath, destDateset, destDateset.length, 1024, false); + assertIsFile(destFilePath); + + boolean rename = rename(srcDir, destDir); + Path renamedSrc = new Path(destDir, sourceSubdir); + assertIsFile(destFilePath); + assertIsDirectory(renamedSrc); + ContractTestUtils.verifyFileContents(fs, destFilePath, destDateset); + assertTrue("rename returned false though the contents were copied", rename); + } + + @Test + public void testRenameFileNonexistentDir() throws Throwable { + describe("rename a file into a new file in the same directory"); + Path renameSrc = path("testRenameSrc"); + Path renameTarget = path("subdir/testRenameTarget"); + byte[] data = dataset(256, 'a', 'z'); + writeDataset(getFileSystem(), renameSrc, data, data.length, 1024 * 1024, + false); + boolean renameCreatesDestDirs = isSupported(RENAME_CREATES_DEST_DIRS); + + try { + boolean rename = rename(renameSrc, renameTarget); + if (renameCreatesDestDirs) { + assertTrue(rename); + ContractTestUtils.verifyFileContents(getFileSystem(), renameTarget, data); + } else { + assertFalse(rename); + ContractTestUtils.verifyFileContents(getFileSystem(), renameSrc, data); + } + } catch (FileNotFoundException e) { + // allowed unless that rename flag is set + assertFalse(renameCreatesDestDirs); + } + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractRootDirectoryTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractRootDirectoryTest.java new file mode 100644 index 00000000000..83d9143e77d --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractRootDirectoryTest.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; + +import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile; +import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; + +/** + * This class does things to the root directory. + * Only subclass this for tests against transient filesystems where + * you don't care about the data. + */ +public abstract class AbstractContractRootDirectoryTest extends AbstractFSContractTestBase { + private static final Logger LOG = + LoggerFactory.getLogger(AbstractContractRootDirectoryTest.class); + + @Override + public void setup() throws Exception { + super.setup(); + skipIfUnsupported(TEST_ROOT_TESTS_ENABLED); + } + + @Test + public void testMkDirDepth1() throws Throwable { + FileSystem fs = getFileSystem(); + Path dir = new Path("/testmkdirdepth1"); + assertPathDoesNotExist("directory already exists", dir); + fs.mkdirs(dir); + ContractTestUtils.assertIsDirectory(getFileSystem(), dir); + assertPathExists("directory already exists", dir); + assertDeleted(dir, true); + } + + @Test + public void testRmEmptyRootDirNonRecursive() throws Throwable { + //extra sanity checks here to avoid support calls about complete loss of data + skipIfUnsupported(TEST_ROOT_TESTS_ENABLED); + Path root = new Path("/"); + ContractTestUtils.assertIsDirectory(getFileSystem(), root); + boolean deleted = getFileSystem().delete(root, true); + LOG.info("rm / of empty dir result is {}", deleted); + ContractTestUtils.assertIsDirectory(getFileSystem(), root); + } + + @Test + public void testRmNonEmptyRootDirNonRecursive() throws Throwable { + //extra sanity checks here to avoid support calls about complete loss of data + skipIfUnsupported(TEST_ROOT_TESTS_ENABLED); + Path root = new Path("/"); + String touchfile = "/testRmNonEmptyRootDirNonRecursive"; + Path file = new Path(touchfile); + ContractTestUtils.touch(getFileSystem(), file); + ContractTestUtils.assertIsDirectory(getFileSystem(), root); + try { + boolean deleted = getFileSystem().delete(root, false); + fail("non recursive delete should have raised an exception," + + " but completed with exit code " + deleted); + } catch (IOException e) { + //expected + handleExpectedException(e); + } finally { + getFileSystem().delete(file, false); + } + ContractTestUtils.assertIsDirectory(getFileSystem(), root); + } + + @Test + public void testRmRootRecursive() throws Throwable { + //extra sanity checks here to avoid support calls about complete loss of data + skipIfUnsupported(TEST_ROOT_TESTS_ENABLED); + Path root = new Path("/"); + ContractTestUtils.assertIsDirectory(getFileSystem(), root); + Path file = new Path("/testRmRootRecursive"); + ContractTestUtils.touch(getFileSystem(), file); + boolean deleted = getFileSystem().delete(root, true); + ContractTestUtils.assertIsDirectory(getFileSystem(), root); + LOG.info("rm -rf / result is {}", deleted); + if (deleted) { + assertPathDoesNotExist("expected file to be deleted", file); + } else { + assertPathExists("expected file to be preserved", file);; + } + } + + @Test + public void testCreateFileOverRoot() throws Throwable { + Path root = new Path("/"); + byte[] dataset = dataset(1024, ' ', 'z'); + try { + createFile(getFileSystem(), root, false, dataset); + fail("expected an exception, got a file created over root: " + ls(root)); + } catch (IOException e) { + //expected + handleExpectedException(e); + } + assertIsDirectory(root); + } + +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSeekTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSeekTest.java new file mode 100644 index 00000000000..4a0560ec256 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSeekTest.java @@ -0,0 +1,348 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IOUtils; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.EOFException; +import java.io.IOException; +import java.util.Random; + +import static org.apache.hadoop.fs.contract.ContractTestUtils.cleanup; +import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile; +import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; +import static org.apache.hadoop.fs.contract.ContractTestUtils.touch; +import static org.apache.hadoop.fs.contract.ContractTestUtils.verifyRead; + +/** + * Test Seek operations + */ +public abstract class AbstractContractSeekTest extends AbstractFSContractTestBase { + private static final Logger LOG = + LoggerFactory.getLogger(AbstractContractSeekTest.class); + + public static final int DEFAULT_RANDOM_SEEK_COUNT = 100; + + private Path testPath; + private Path smallSeekFile; + private Path zeroByteFile; + private FSDataInputStream instream; + + @Override + public void setup() throws Exception { + super.setup(); + skipIfUnsupported(SUPPORTS_SEEK); + //delete the test directory + testPath = getContract().getTestPath(); + smallSeekFile = path("seekfile.txt"); + zeroByteFile = path("zero.txt"); + byte[] block = dataset(TEST_FILE_LEN, 0, 255); + //this file now has a simple rule: offset => value + createFile(getFileSystem(), smallSeekFile, false, block); + touch(getFileSystem(), zeroByteFile); + } + + @Override + protected Configuration createConfiguration() { + Configuration conf = super.createConfiguration(); + conf.setInt(CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY, 4096); + return conf; + } + + @Override + public void teardown() throws Exception { + IOUtils.closeStream(instream); + instream = null; + super.teardown(); + } + + @Test + public void testSeekZeroByteFile() throws Throwable { + describe("seek and read a 0 byte file"); + instream = getFileSystem().open(zeroByteFile); + assertEquals(0, instream.getPos()); + //expect initial read to fai; + int result = instream.read(); + assertMinusOne("initial byte read", result); + byte[] buffer = new byte[1]; + //expect that seek to 0 works + instream.seek(0); + //reread, expect same exception + result = instream.read(); + assertMinusOne("post-seek byte read", result); + result = instream.read(buffer, 0, 1); + assertMinusOne("post-seek buffer read", result); + } + + @Test + public void testBlockReadZeroByteFile() throws Throwable { + describe("do a block read on a 0 byte file"); + instream = getFileSystem().open(zeroByteFile); + assertEquals(0, instream.getPos()); + //expect that seek to 0 works + byte[] buffer = new byte[1]; + int result = instream.read(buffer, 0, 1); + assertMinusOne("block read zero byte file", result); + } + + /** + * Seek and read on a closed file. + * Some filesystems let callers seek on a closed file -these must + * still fail on the subsequent reads. + * @throws Throwable + */ + @Test + public void testSeekReadClosedFile() throws Throwable { + boolean supportsSeekOnClosedFiles = isSupported(SUPPORTS_SEEK_ON_CLOSED_FILE); + + instream = getFileSystem().open(smallSeekFile); + getLog().debug( + "Stream is of type " + instream.getClass().getCanonicalName()); + instream.close(); + try { + instream.seek(0); + if (!supportsSeekOnClosedFiles) { + fail("seek succeeded on a closed stream"); + } + } catch (IOException e) { + //expected a closed file + } + try { + int data = instream.available(); + fail("read() succeeded on a closed stream, got " + data); + } catch (IOException e) { + //expected a closed file + } + try { + int data = instream.read(); + fail("read() succeeded on a closed stream, got " + data); + } catch (IOException e) { + //expected a closed file + } + try { + byte[] buffer = new byte[1]; + int result = instream.read(buffer, 0, 1); + fail("read(buffer, 0, 1) succeeded on a closed stream, got " + result); + } catch (IOException e) { + //expected a closed file + } + //what position does a closed file have? + try { + long offset = instream.getPos(); + } catch (IOException e) { + // its valid to raise error here; but the test is applied to make + // sure there's no other exception like an NPE. + + } + //and close again + instream.close(); + } + + @Test + public void testNegativeSeek() throws Throwable { + instream = getFileSystem().open(smallSeekFile); + assertEquals(0, instream.getPos()); + try { + instream.seek(-1); + long p = instream.getPos(); + LOG.warn("Seek to -1 returned a position of " + p); + int result = instream.read(); + fail( + "expected an exception, got data " + result + " at a position of " + p); + } catch (EOFException e) { + //bad seek -expected + handleExpectedException(e); + } catch (IOException e) { + //bad seek -expected, but not as preferred as an EOFException + handleRelaxedException("a negative seek", "EOFException", e); + } + assertEquals(0, instream.getPos()); + } + + @Test + public void testSeekFile() throws Throwable { + describe("basic seek operations"); + instream = getFileSystem().open(smallSeekFile); + assertEquals(0, instream.getPos()); + //expect that seek to 0 works + instream.seek(0); + int result = instream.read(); + assertEquals(0, result); + assertEquals(1, instream.read()); + assertEquals(2, instream.getPos()); + assertEquals(2, instream.read()); + assertEquals(3, instream.getPos()); + instream.seek(128); + assertEquals(128, instream.getPos()); + assertEquals(128, instream.read()); + instream.seek(63); + assertEquals(63, instream.read()); + } + + @Test + public void testSeekAndReadPastEndOfFile() throws Throwable { + describe("verify that reading past the last bytes in the file returns -1"); + instream = getFileSystem().open(smallSeekFile); + assertEquals(0, instream.getPos()); + //expect that seek to 0 works + //go just before the end + instream.seek(TEST_FILE_LEN - 2); + assertTrue("Premature EOF", instream.read() != -1); + assertTrue("Premature EOF", instream.read() != -1); + assertMinusOne("read past end of file", instream.read()); + } + + @Test + public void testSeekPastEndOfFileThenReseekAndRead() throws Throwable { + describe("do a seek past the EOF, then verify the stream recovers"); + instream = getFileSystem().open(smallSeekFile); + //go just before the end. This may or may not fail; it may be delayed until the + //read + boolean canSeekPastEOF = + !getContract().isSupported(ContractOptions.REJECTS_SEEK_PAST_EOF, true); + try { + instream.seek(TEST_FILE_LEN + 1); + //if this doesn't trigger, then read() is expected to fail + assertMinusOne("read after seeking past EOF", instream.read()); + } catch (EOFException e) { + //This is an error iff the FS claims to be able to seek past the EOF + if (canSeekPastEOF) { + //a failure wasn't expected + throw e; + } + handleExpectedException(e); + } catch (IOException e) { + //This is an error iff the FS claims to be able to seek past the EOF + if (canSeekPastEOF) { + //a failure wasn't expected + throw e; + } + handleRelaxedException("a seek past the end of the file", + "EOFException", e); + } + //now go back and try to read from a valid point in the file + instream.seek(1); + assertTrue("Premature EOF", instream.read() != -1); + } + + /** + * Seek round a file bigger than IO buffers + * @throws Throwable + */ + @Test + public void testSeekBigFile() throws Throwable { + describe("Seek round a large file and verify the bytes are what is expected"); + Path testSeekFile = path("bigseekfile.txt"); + byte[] block = dataset(65536, 0, 255); + createFile(getFileSystem(), testSeekFile, false, block); + instream = getFileSystem().open(testSeekFile); + assertEquals(0, instream.getPos()); + //expect that seek to 0 works + instream.seek(0); + int result = instream.read(); + assertEquals(0, result); + assertEquals(1, instream.read()); + assertEquals(2, instream.read()); + + //do seek 32KB ahead + instream.seek(32768); + assertEquals("@32768", block[32768], (byte) instream.read()); + instream.seek(40000); + assertEquals("@40000", block[40000], (byte) instream.read()); + instream.seek(8191); + assertEquals("@8191", block[8191], (byte) instream.read()); + instream.seek(0); + assertEquals("@0", 0, (byte) instream.read()); + } + + @Test + public void testPositionedBulkReadDoesntChangePosition() throws Throwable { + describe( + "verify that a positioned read does not change the getPos() value"); + Path testSeekFile = path("bigseekfile.txt"); + byte[] block = dataset(65536, 0, 255); + createFile(getFileSystem(), testSeekFile, false, block); + instream = getFileSystem().open(testSeekFile); + instream.seek(39999); + assertTrue(-1 != instream.read()); + assertEquals(40000, instream.getPos()); + + byte[] readBuffer = new byte[256]; + instream.read(128, readBuffer, 0, readBuffer.length); + //have gone back + assertEquals(40000, instream.getPos()); + //content is the same too + assertEquals("@40000", block[40000], (byte) instream.read()); + //now verify the picked up data + for (int i = 0; i < 256; i++) { + assertEquals("@" + i, block[i + 128], readBuffer[i]); + } + } + + /** + * Lifted from TestLocalFileSystem: + * Regression test for HADOOP-9307: BufferedFSInputStream returning + * wrong results after certain sequences of seeks and reads. + */ + @Test + public void testRandomSeeks() throws Throwable { + int limit = getContract().getLimit(TEST_RANDOM_SEEK_COUNT, + DEFAULT_RANDOM_SEEK_COUNT); + describe("Testing " + limit + " random seeks"); + int filesize = 10 * 1024; + byte[] buf = dataset(filesize, 0, 255); + Path randomSeekFile = path("testrandomseeks.bin"); + createFile(getFileSystem(), randomSeekFile, false, buf); + Random r = new Random(); + FSDataInputStream stm = getFileSystem().open(randomSeekFile); + + // Record the sequence of seeks and reads which trigger a failure. + int[] seeks = new int[10]; + int[] reads = new int[10]; + try { + for (int i = 0; i < limit; i++) { + int seekOff = r.nextInt(buf.length); + int toRead = r.nextInt(Math.min(buf.length - seekOff, 32000)); + + seeks[i % seeks.length] = seekOff; + reads[i % reads.length] = toRead; + verifyRead(stm, buf, seekOff, toRead); + } + } catch (AssertionError afe) { + StringBuilder sb = new StringBuilder(); + sb.append("Sequence of actions:\n"); + for (int j = 0; j < seeks.length; j++) { + sb.append("seek @ ").append(seeks[j]).append(" ") + .append("read ").append(reads[j]).append("\n"); + } + LOG.error(sb.toString()); + throw afe; + } finally { + stm.close(); + } + } + +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractFSContract.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractFSContract.java new file mode 100644 index 00000000000..d3dafe974a5 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractFSContract.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.junit.Assert; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; + +/** + * Class representing a filesystem contract that a filesystem + * implementation is expected implement. + * + * Part of this contract class is to allow FS implementations to + * provide specific opt outs and limits, so that tests can be + * skip unsupported features (e.g. case sensitivity tests), + * dangerous operations (e.g. trying to delete the root directory), + * and limit filesize and other numeric variables for scale tests + */ +public abstract class AbstractFSContract extends Configured { + private static final Logger LOG = + LoggerFactory.getLogger(AbstractFSContract.class); + + private boolean enabled = true; + + + /** + * Constructor: loads the authentication keys if found + * @param conf configuration to work with + */ + protected AbstractFSContract(Configuration conf) { + super(conf); + if (maybeAddConfResource(ContractOptions.CONTRACT_OPTIONS_RESOURCE)) { + LOG.debug("Loaded authentication keys from {}", ContractOptions.CONTRACT_OPTIONS_RESOURCE); + } else { + LOG.debug("Not loaded: {}", ContractOptions.CONTRACT_OPTIONS_RESOURCE); + } + } + + /** + * Any initialisation logic can go here + * @throws IOException IO problems + */ + public void init() throws IOException { + + } + + /** + * Add a configuration resource to this instance's configuration + * @param resource resource reference + * @throws AssertionError if the resource was not found. + */ + protected void addConfResource(String resource) { + boolean found = maybeAddConfResource(resource); + Assert.assertTrue("Resource not found " + resource, found); + } + + /** + * Add a configuration resource to this instance's configuration, + * return true if the resource was found + * @param resource resource reference + */ + protected boolean maybeAddConfResource(String resource) { + URL url = this.getClass().getClassLoader().getResource(resource); + boolean found = url != null; + if (found) { + getConf().addResource(resource); + } + return found; + } + + + /** + * Get the FS from a URI. The default implementation just retrieves + * it from the norrmal FileSystem factory/cache, with the local configuration + * @param uri URI of FS + * @return the filesystem + * @throws IOException IO problems + */ + public FileSystem getFileSystem(URI uri) throws IOException { + return FileSystem.get(uri, getConf()); + } + + /** + * Get the filesystem for these tests + * @return the test fs + * @throws IOException IO problems + */ + public abstract FileSystem getTestFileSystem() throws IOException; + + /** + * Get the scheme of this FS + * @return the scheme this FS supports + */ + public abstract String getScheme(); + + /** + * Return the path string for tests, e.g. file:///tmp + * @return a path in the test FS + */ + public abstract Path getTestPath(); + + /** + * Boolean to indicate whether or not the contract test are enabled + * for this test run. + * @return true if the tests can be run. + */ + public boolean isEnabled() { + return enabled; + } + + /** + * Boolean to indicate whether or not the contract test are enabled + * for this test run. + * @param enabled flag which must be true if the tests can be run. + */ + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + /** + * Query for a feature being supported. This may include a probe for the feature + * + * @param feature feature to query + * @param defval default value + * @return true if the feature is supported + * @throws IOException IO problems + */ + public boolean isSupported(String feature, boolean defval) { + return getConf().getBoolean(getConfKey(feature), defval); + } + + /** + * Query for a feature's limit. This may include a probe for the feature + * + * @param feature feature to query + * @param defval default value + * @return true if the feature is supported + * @throws IOException IO problems + */ + public int getLimit(String feature, int defval) { + return getConf().getInt(getConfKey(feature), defval); + } + + public String getOption(String feature, String defval) { + return getConf().get(getConfKey(feature), defval); + } + + /** + * Build a configuration key + * @param feature feature to query + * @return the configuration key base with the feature appended + */ + public String getConfKey(String feature) { + return ContractOptions.FS_CONTRACT_KEY + feature; + } + + /** + * Create a URI off the scheme + * @param path path of URI + * @return a URI + * @throws IOException if the URI could not be created + */ + protected URI toURI(String path) throws IOException { + try { + return new URI(getScheme(),path, null); + } catch (URISyntaxException e) { + throw new IOException(e.toString() + " with " + path, e); + } + } + + @Override + public String toString() { + return "FSContract for " + getScheme(); + } + +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractFSContractTestBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractFSContractTestBase.java new file mode 100644 index 00000000000..a000ec8ed55 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractFSContractTestBase.java @@ -0,0 +1,363 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.internal.AssumptionViolatedException; +import org.junit.rules.Timeout; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URI; + +import static org.apache.hadoop.fs.contract.ContractTestUtils.cleanup; +import static org.apache.hadoop.fs.contract.ContractTestUtils.skip; + +/** + * This is the base class for all the contract tests + */ +public abstract class AbstractFSContractTestBase extends Assert + implements ContractOptions { + + private static final Logger LOG = + LoggerFactory.getLogger(AbstractFSContractTestBase.class); + + /** + * Length of files to work with: {@value} + */ + public static final int TEST_FILE_LEN = 1024; + + /** + * standard test timeout: {@value} + */ + public static final int DEFAULT_TEST_TIMEOUT = 180 * 1000; + + /** + * The FS contract used for these tets + */ + private AbstractFSContract contract; + + /** + * The test filesystem extracted from it + */ + private FileSystem fileSystem; + + /** + * The path for tests + */ + private Path testPath; + + /** + * This must be implemented by all instantiated test cases + * -provide the FS contract + * @return the FS contract + */ + protected abstract AbstractFSContract createContract(Configuration conf); + + /** + * Get the contract + * @return the contract, which will be non-null once the setup operation has + * succeeded + */ + protected AbstractFSContract getContract() { + return contract; + } + + /** + * Get the filesystem created in startup + * @return the filesystem to use for tests + */ + public FileSystem getFileSystem() { + return fileSystem; + } + + /** + * Get the log of the base class + * @return a logger + */ + public static Logger getLog() { + return LOG; + } + + /** + * Skip a test if a feature is unsupported in this FS + * @param feature feature to look for + * @throws IOException IO problem + */ + protected void skipIfUnsupported(String feature) throws IOException { + if (!isSupported(feature)) { + skip("Skipping as unsupported feature: " + feature); + } + } + + /** + * Is a feature supported? + * @param feature feature + * @return true iff the feature is supported + * @throws IOException IO problems + */ + protected boolean isSupported(String feature) throws IOException { + return contract.isSupported(feature, false); + } + + /** + * Include at the start of tests to skip them if the FS is not enabled. + */ + protected void assumeEnabled() { + if (!contract.isEnabled()) + throw new AssumptionViolatedException("test cases disabled for " + contract); + } + + /** + * Create a configuration. May be overridden by tests/instantiations + * @return a configuration + */ + protected Configuration createConfiguration() { + return new Configuration(); + } + + /** + * Set the timeout for every test + */ + @Rule + public Timeout testTimeout = new Timeout(getTestTimeoutMillis()); + + /** + * Option for tests to override the default timeout value + * @return the current test timeout + */ + protected int getTestTimeoutMillis() { + return DEFAULT_TEST_TIMEOUT; + } + + + /** + * Setup: create the contract then init it + * @throws Exception on any failure + */ + @Before + public void setup() throws Exception { + contract = createContract(createConfiguration()); + contract.init(); + //skip tests if they aren't enabled + assumeEnabled(); + //extract the test FS + fileSystem = contract.getTestFileSystem(); + assertNotNull("null filesystem", fileSystem); + URI fsURI = fileSystem.getUri(); + LOG.info("Test filesystem = {} implemented by {}", + fsURI, fileSystem); + //sanity check to make sure that the test FS picked up really matches + //the scheme chosen. This is to avoid defaulting back to the localFS + //which would be drastic for root FS tests + assertEquals("wrong filesystem of " + fsURI, + contract.getScheme(), fsURI.getScheme()); + //create the test path + testPath = getContract().getTestPath(); + mkdirs(testPath); + } + + /** + * Teardown + * @throws Exception on any failure + */ + @After + public void teardown() throws Exception { + deleteTestDirInTeardown(); + } + + /** + * Delete the test dir in the per-test teardown + * @throws IOException + */ + protected void deleteTestDirInTeardown() throws IOException { + cleanup("TEARDOWN", getFileSystem(), testPath); + } + + /** + * Create a path under the test path provided by + * the FS contract + * @param filepath path string in + * @return a path qualified by the test filesystem + * @throws IOException IO problems + */ + protected Path path(String filepath) throws IOException { + return getFileSystem().makeQualified( + new Path(getContract().getTestPath(), filepath)); + } + + /** + * Take a simple path like "/something" and turn it into + * a qualified path against the test FS + * @param filepath path string in + * @return a path qualified by the test filesystem + * @throws IOException IO problems + */ + protected Path absolutepath(String filepath) throws IOException { + return getFileSystem().makeQualified(new Path(filepath)); + } + + /** + * List a path in the test FS + * @param path path to list + * @return the contents of the path/dir + * @throws IOException IO problems + */ + protected String ls(Path path) throws IOException { + return ContractTestUtils.ls(fileSystem, path); + } + + /** + * Describe a test. This is a replacement for javadocs + * where the tests role is printed in the log output + * @param text description + */ + protected void describe(String text) { + LOG.info(text); + } + + /** + * Handle the outcome of an operation not being the strictest + * exception desired, but one that, while still within the boundary + * of the contract, is a bit looser. + * + * If the FS contract says that they support the strictest exceptions, + * that is what they must return, and the exception here is rethrown + * @param action Action + * @param expectedException what was expected + * @param e exception that was received + */ + protected void handleRelaxedException(String action, + String expectedException, + Exception e) throws Exception { + if (getContract().isSupported(SUPPORTS_STRICT_EXCEPTIONS, false)) { + throw e; + } + LOG.warn("The expected exception {} was not the exception class" + + " raised on {}: {}", action , e.getClass(), expectedException, e); + } + + /** + * Handle expected exceptions through logging and/or other actions + * @param e exception raised. + */ + protected void handleExpectedException(Exception e) { + getLog().debug("expected :{}" ,e, e); + } + + /** + * assert that a path exists + * @param message message to use in an assertion + * @param path path to probe + * @throws IOException IO problems + */ + public void assertPathExists(String message, Path path) throws IOException { + ContractTestUtils.assertPathExists(fileSystem, message, path); + } + + /** + * assert that a path does not + * @param message message to use in an assertion + * @param path path to probe + * @throws IOException IO problems + */ + public void assertPathDoesNotExist(String message, Path path) throws + IOException { + ContractTestUtils.assertPathDoesNotExist(fileSystem, message, path); + } + + /** + * Assert that a file exists and whose {@link FileStatus} entry + * declares that this is a file and not a symlink or directory. + * + * @param filename name of the file + * @throws IOException IO problems during file operations + */ + protected void assertIsFile(Path filename) throws IOException { + ContractTestUtils.assertIsFile(fileSystem, filename); + } + + /** + * Assert that a file exists and whose {@link FileStatus} entry + * declares that this is a file and not a symlink or directory. + * + * @param path name of the file + * @throws IOException IO problems during file operations + */ + protected void assertIsDirectory(Path path) throws IOException { + ContractTestUtils.assertIsDirectory(fileSystem, path); + } + + + /** + * Assert that a file exists and whose {@link FileStatus} entry + * declares that this is a file and not a symlink or directory. + * + * @throws IOException IO problems during file operations + */ + protected void mkdirs(Path path) throws IOException { + assertTrue("Failed to mkdir " + path, fileSystem.mkdirs(path)); + } + + /** + * Assert that a delete succeeded + * @param path path to delete + * @param recursive recursive flag + * @throws IOException IO problems + */ + protected void assertDeleted(Path path, boolean recursive) throws + IOException { + ContractTestUtils.assertDeleted(fileSystem, path, recursive); + } + + /** + * Assert that the result value == -1; which implies + * that a read was successful + * @param text text to include in a message (usually the operation) + * @param result read result to validate + */ + protected void assertMinusOne(String text, int result) { + assertEquals(text + " wrong read result " + result, -1, result); + } + + boolean rename(Path src, Path dst) throws IOException { + return getFileSystem().rename(src, dst); + } + + protected String generateAndLogErrorListing(Path src, Path dst) throws + IOException { + FileSystem fs = getFileSystem(); + getLog().error( + "src dir " + ContractTestUtils.ls(fs, src.getParent())); + String destDirLS = ContractTestUtils.ls(fs, dst.getParent()); + if (fs.isDirectory(dst)) { + //include the dir into the listing + destDirLS = destDirLS + "\n" + ContractTestUtils.ls(fs, dst); + } + return destDirLS; + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractOptions.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractOptions.java new file mode 100644 index 00000000000..61279b02ee8 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractOptions.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract; + +/** + * Options for contract tests: keys for FS-specific values, + * defaults. + */ +public interface ContractOptions { + + + /** + * name of the (optional) resource containing filesystem binding keys : {@value} + * If found, it it will be loaded + */ + String CONTRACT_OPTIONS_RESOURCE = "contract-test-options.xml"; + + /** + * Prefix for all contract keys in the configuration files + */ + String FS_CONTRACT_KEY = "fs.contract."; + + /** + * Is a filesystem case sensitive. + * Some of the filesystems that say "no" here may mean + * that it varies from platform to platform -the localfs being the key + * example. + */ + String IS_CASE_SENSITIVE = "is-case-sensitive"; + + /** + * Blobstore flag. Implies it's not a real directory tree and + * consistency is below that which Hadoop expects + */ + String IS_BLOBSTORE = "is-blobstore"; + + /** + * Flag to indicate that the FS can rename into directories that + * don't exist, creating them as needed. + * @{value} + */ + String RENAME_CREATES_DEST_DIRS = "rename-creates-dest-dirs"; + + /** + * Flag to indicate that the FS does not follow the rename contract -and + * instead only returns false on a failure. + * @{value} + */ + String RENAME_OVERWRITES_DEST = "rename-overwrites-dest"; + + /** + * Flag to indicate that the FS returns false if the destination exists + * @{value} + */ + String RENAME_RETURNS_FALSE_IF_DEST_EXISTS = + "rename-returns-false-if-dest-exists"; + + /** + * Flag to indicate that the FS returns false on a rename + * if the source is missing + * @{value} + */ + String RENAME_RETURNS_FALSE_IF_SOURCE_MISSING = + "rename-returns-false-if-source-missing"; + + /** + * Flag to indicate that append is supported + * @{value} + */ + String SUPPORTS_APPEND = "supports-append"; + + /** + * Flag to indicate that renames are atomic + * @{value} + */ + String SUPPORTS_ATOMIC_RENAME = "supports-atomic-rename"; + + /** + * Flag to indicate that directory deletes are atomic + * @{value} + */ + String SUPPORTS_ATOMIC_DIRECTORY_DELETE = "supports-atomic-directory-delete"; + + /** + * Does the FS support multiple block locations? + * @{value} + */ + String SUPPORTS_BLOCK_LOCALITY = "supports-block-locality"; + + /** + * Does the FS support the concat() operation? + * @{value} + */ + String SUPPORTS_CONCAT = "supports-concat"; + + /** + * Is seeking supported at all? + * @{value} + */ + String SUPPORTS_SEEK = "supports-seek"; + + /** + * Is seeking past the EOF allowed? + * @{value} + */ + String REJECTS_SEEK_PAST_EOF = "rejects-seek-past-eof"; + + /** + * Is seeking on a closed file supported? Some filesystems only raise an + * exception later, when trying to read. + * @{value} + */ + String SUPPORTS_SEEK_ON_CLOSED_FILE = "supports-seek-on-closed-file"; + + /** + * Flag to indicate that this FS expects to throw the strictest + * exceptions it can, not generic IOEs, which, if returned, + * must be rejected. + * @{value} + */ + String SUPPORTS_STRICT_EXCEPTIONS = "supports-strict-exceptions"; + + /** + * Are unix permissions + * @{value} + */ + String SUPPORTS_UNIX_PERMISSIONS = "supports-unix-permissions"; + + /** + * Maximum path length + * @{value} + */ + String MAX_PATH_ = "max-path"; + + /** + * Maximum filesize: 0 or -1 for no limit + * @{value} + */ + String MAX_FILESIZE = "max-filesize"; + + /** + * Flag to indicate that tests on the root directories of a filesystem/ + * object store are permitted + * @{value} + */ + String TEST_ROOT_TESTS_ENABLED = "test.root-tests-enabled"; + + /** + * Limit for #of random seeks to perform. + * Keep low for remote filesystems for faster tests + */ + String TEST_RANDOM_SEEK_COUNT = "test.random-seek-count"; + +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java new file mode 100644 index 00000000000..cd9cc1ba154 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java @@ -0,0 +1,759 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract; + +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.junit.Assert; +import org.junit.internal.AssumptionViolatedException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.EOFException; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.Arrays; +import java.util.Properties; + +/** + * Utilities used across test cases + */ +public class ContractTestUtils extends Assert { + + private static final Logger LOG = + LoggerFactory.getLogger(ContractTestUtils.class); + + public static final String IO_FILE_BUFFER_SIZE = "io.file.buffer.size"; + + /** + * Assert that a property in the property set matches the expected value + * @param props property set + * @param key property name + * @param expected expected value. If null, the property must not be in the set + */ + public static void assertPropertyEquals(Properties props, + String key, + String expected) { + String val = props.getProperty(key); + if (expected == null) { + assertNull("Non null property " + key + " = " + val, val); + } else { + assertEquals("property " + key + " = " + val, + expected, + val); + } + } + + /** + * + * Write a file and read it in, validating the result. Optional flags control + * whether file overwrite operations should be enabled, and whether the + * file should be deleted afterwards. + * + * If there is a mismatch between what was written and what was expected, + * a small range of bytes either side of the first error are logged to aid + * diagnosing what problem occurred -whether it was a previous file + * or a corrupting of the current file. This assumes that two + * sequential runs to the same path use datasets with different character + * moduli. + * + * @param fs filesystem + * @param path path to write to + * @param len length of data + * @param overwrite should the create option allow overwrites? + * @param delete should the file be deleted afterwards? -with a verification + * that it worked. Deletion is not attempted if an assertion has failed + * earlier -it is not in a finally{} block. + * @throws IOException IO problems + */ + public static void writeAndRead(FileSystem fs, + Path path, + byte[] src, + int len, + int blocksize, + boolean overwrite, + boolean delete) throws IOException { + fs.mkdirs(path.getParent()); + + writeDataset(fs, path, src, len, blocksize, overwrite); + + byte[] dest = readDataset(fs, path, len); + + compareByteArrays(src, dest, len); + + if (delete) { + rejectRootOperation(path); + boolean deleted = fs.delete(path, false); + assertTrue("Deleted", deleted); + assertPathDoesNotExist(fs, "Cleanup failed", path); + } + } + + /** + * Write a file. + * Optional flags control + * whether file overwrite operations should be enabled + * @param fs filesystem + * @param path path to write to + * @param len length of data + * @param overwrite should the create option allow overwrites? + * @throws IOException IO problems + */ + public static void writeDataset(FileSystem fs, + Path path, + byte[] src, + int len, + int buffersize, + boolean overwrite) throws IOException { + assertTrue( + "Not enough data in source array to write " + len + " bytes", + src.length >= len); + FSDataOutputStream out = fs.create(path, + overwrite, + fs.getConf() + .getInt(IO_FILE_BUFFER_SIZE, + 4096), + (short) 1, + buffersize); + out.write(src, 0, len); + out.close(); + assertFileHasLength(fs, path, len); + } + + /** + * Read the file and convert to a byte dataset. + * This implements readfully internally, so that it will read + * in the file without ever having to seek() + * @param fs filesystem + * @param path path to read from + * @param len length of data to read + * @return the bytes + * @throws IOException IO problems + */ + public static byte[] readDataset(FileSystem fs, Path path, int len) + throws IOException { + FSDataInputStream in = fs.open(path); + byte[] dest = new byte[len]; + int offset =0; + int nread = 0; + try { + while (nread < len) { + int nbytes = in.read(dest, offset + nread, len - nread); + if (nbytes < 0) { + throw new EOFException("End of file reached before reading fully."); + } + nread += nbytes; + } + } finally { + in.close(); + } + return dest; + } + + /** + * Read a file, verify its length and contents match the expected array + * @param fs filesystem + * @param path path to file + * @param original original dataset + * @throws IOException IO Problems + */ + public static void verifyFileContents(FileSystem fs, + Path path, + byte[] original) throws IOException { + FileStatus stat = fs.getFileStatus(path); + String statText = stat.toString(); + assertTrue("not a file " + statText, stat.isFile()); + assertEquals("wrong length " + statText, original.length, stat.getLen()); + byte[] bytes = readDataset(fs, path, original.length); + compareByteArrays(original,bytes,original.length); + } + + /** + * Verify that the read at a specific offset in a stream + * matches that expected + * @param stm stream + * @param fileContents original file contents + * @param seekOff seek offset + * @param toRead number of bytes to read + * @throws IOException IO problems + */ + public static void verifyRead(FSDataInputStream stm, byte[] fileContents, + int seekOff, int toRead) throws IOException { + byte[] out = new byte[toRead]; + stm.seek(seekOff); + stm.readFully(out); + byte[] expected = Arrays.copyOfRange(fileContents, seekOff, + seekOff + toRead); + compareByteArrays(expected, out,toRead); + } + + /** + * Assert that tthe array original[0..len] and received[] are equal. + * A failure triggers the logging of the bytes near where the first + * difference surfaces. + * @param original source data + * @param received actual + * @param len length of bytes to compare + */ + public static void compareByteArrays(byte[] original, + byte[] received, + int len) { + assertEquals("Number of bytes read != number written", + len, received.length); + int errors = 0; + int first_error_byte = -1; + for (int i = 0; i < len; i++) { + if (original[i] != received[i]) { + if (errors == 0) { + first_error_byte = i; + } + errors++; + } + } + + if (errors > 0) { + String message = String.format(" %d errors in file of length %d", + errors, len); + LOG.warn(message); + // the range either side of the first error to print + // this is a purely arbitrary number, to aid user debugging + final int overlap = 10; + for (int i = Math.max(0, first_error_byte - overlap); + i < Math.min(first_error_byte + overlap, len); + i++) { + byte actual = received[i]; + byte expected = original[i]; + String letter = toChar(actual); + String line = String.format("[%04d] %2x %s\n", i, actual, letter); + if (expected != actual) { + line = String.format("[%04d] %2x %s -expected %2x %s\n", + i, + actual, + letter, + expected, + toChar(expected)); + } + LOG.warn(line); + } + fail(message); + } + } + + /** + * Convert a byte to a character for printing. If the + * byte value is < 32 -and hence unprintable- the byte is + * returned as a two digit hex value + * @param b byte + * @return the printable character string + */ + public static String toChar(byte b) { + if (b >= 0x20) { + return Character.toString((char) b); + } else { + return String.format("%02x", b); + } + } + + /** + * Convert a buffer to a string, character by character + * @param buffer input bytes + * @return a string conversion + */ + public static String toChar(byte[] buffer) { + StringBuilder builder = new StringBuilder(buffer.length); + for (byte b : buffer) { + builder.append(toChar(b)); + } + return builder.toString(); + } + + public static byte[] toAsciiByteArray(String s) { + char[] chars = s.toCharArray(); + int len = chars.length; + byte[] buffer = new byte[len]; + for (int i = 0; i < len; i++) { + buffer[i] = (byte) (chars[i] & 0xff); + } + return buffer; + } + + /** + * Cleanup at the end of a test run + * @param action action triggering the operation (for use in logging) + * @param fileSystem filesystem to work with. May be null + * @param cleanupPath path to delete as a string + */ + public static void cleanup(String action, + FileSystem fileSystem, + String cleanupPath) { + if (fileSystem == null) { + return; + } + Path path = new Path(cleanupPath).makeQualified(fileSystem.getUri(), + fileSystem.getWorkingDirectory()); + cleanup(action, fileSystem, path); + } + + /** + * Cleanup at the end of a test run + * @param action action triggering the operation (for use in logging) + * @param fileSystem filesystem to work with. May be null + * @param path path to delete + */ + public static void cleanup(String action, FileSystem fileSystem, Path path) { + noteAction(action); + try { + rm(fileSystem, path, true, false); + } catch (Exception e) { + LOG.error("Error deleting in "+ action + " - " + path + ": " + e, e); + } + } + + /** + * Delete a directory. There's a safety check for operations against the + * root directory -these are intercepted and rejected with an IOException + * unless the allowRootDelete flag is true + * @param fileSystem filesystem to work with. May be null + * @param path path to delete + * @param recursive flag to enable recursive delete + * @param allowRootDelete can the root directory be deleted? + * @throws IOException on any problem. + */ + public static boolean rm(FileSystem fileSystem, + Path path, + boolean recursive, + boolean allowRootDelete) throws + IOException { + if (fileSystem != null) { + rejectRootOperation(path, allowRootDelete); + if (fileSystem.exists(path)) { + return fileSystem.delete(path, recursive); + } + } + return false; + + } + + /** + * Block any operation on the root path. This is a safety check + * @param path path in the filesystem + * @param allowRootOperation can the root directory be manipulated? + * @throws IOException if the operation was rejected + */ + public static void rejectRootOperation(Path path, + boolean allowRootOperation) throws IOException { + if (path.isRoot() && !allowRootOperation) { + throw new IOException("Root directory operation rejected: " + path); + } + } + + /** + * Block any operation on the root path. This is a safety check + * @param path path in the filesystem + * @throws IOException if the operation was rejected + */ + public static void rejectRootOperation(Path path) throws IOException { + rejectRootOperation(path, false); + } + + + public static void noteAction(String action) { + if (LOG.isDebugEnabled()) { + LOG.debug("============== "+ action +" ============="); + } + } + + /** + * downgrade a failure to a message and a warning, then an + * exception for the Junit test runner to mark as failed + * @param message text message + * @param failure what failed + * @throws AssumptionViolatedException always + */ + public static void downgrade(String message, Throwable failure) { + LOG.warn("Downgrading test " + message, failure); + AssumptionViolatedException ave = + new AssumptionViolatedException(failure, null); + throw ave; + } + + /** + * report an overridden test as unsupported + * @param message message to use in the text + * @throws AssumptionViolatedException always + */ + public static void unsupported(String message) { + skip(message); + } + + /** + * report a test has been skipped for some reason + * @param message message to use in the text + * @throws AssumptionViolatedException always + */ + public static void skip(String message) { + LOG.info("Skipping: {}", message); + throw new AssumptionViolatedException(message); + } + + /** + * Fail with an exception that was received + * @param text text to use in the exception + * @param thrown a (possibly null) throwable to init the cause with + * @throws AssertionError with the text and throwable -always + */ + public static void fail(String text, Throwable thrown) { + AssertionError e = new AssertionError(text); + e.initCause(thrown); + throw e; + } + + /** + * Make an assertion about the length of a file + * @param fs filesystem + * @param path path of the file + * @param expected expected length + * @throws IOException on File IO problems + */ + public static void assertFileHasLength(FileSystem fs, Path path, + int expected) throws IOException { + FileStatus status = fs.getFileStatus(path); + assertEquals( + "Wrong file length of file " + path + " status: " + status, + expected, + status.getLen()); + } + + /** + * Assert that a path refers to a directory + * @param fs filesystem + * @param path path of the directory + * @throws IOException on File IO problems + */ + public static void assertIsDirectory(FileSystem fs, + Path path) throws IOException { + FileStatus fileStatus = fs.getFileStatus(path); + assertIsDirectory(fileStatus); + } + + /** + * Assert that a path refers to a directory + * @param fileStatus stats to check + */ + public static void assertIsDirectory(FileStatus fileStatus) { + assertTrue("Should be a directory -but isn't: " + fileStatus, + fileStatus.isDirectory()); + } + + /** + * Write the text to a file, returning the converted byte array + * for use in validating the round trip + * @param fs filesystem + * @param path path of file + * @param text text to write + * @param overwrite should the operation overwrite any existing file? + * @return the read bytes + * @throws IOException on IO problems + */ + public static byte[] writeTextFile(FileSystem fs, + Path path, + String text, + boolean overwrite) throws IOException { + byte[] bytes = new byte[0]; + if (text != null) { + bytes = toAsciiByteArray(text); + } + createFile(fs, path, overwrite, bytes); + return bytes; + } + + /** + * Create a file + * @param fs filesystem + * @param path path to write + * @param overwrite overwrite flag + * @param data source dataset. Can be null + * @throws IOException on any problem + */ + public static void createFile(FileSystem fs, + Path path, + boolean overwrite, + byte[] data) throws IOException { + FSDataOutputStream stream = fs.create(path, overwrite); + if (data != null && data.length > 0) { + stream.write(data); + } + stream.close(); + } + + /** + * Touch a file + * @param fs filesystem + * @param path path + * @throws IOException IO problems + */ + public static void touch(FileSystem fs, + Path path) throws IOException { + createFile(fs, path, true, null); + } + + /** + * Delete a file/dir and assert that delete() returned true + * and that the path no longer exists. This variant rejects + * all operations on root directories + * @param fs filesystem + * @param file path to delete + * @param recursive flag to enable recursive delete + * @throws IOException IO problems + */ + public static void assertDeleted(FileSystem fs, + Path file, + boolean recursive) throws IOException { + assertDeleted(fs, file, recursive, false); + } + + /** + * Delete a file/dir and assert that delete() returned true + * and that the path no longer exists. This variant rejects + * all operations on root directories + * @param fs filesystem + * @param file path to delete + * @param recursive flag to enable recursive delete + * @param allowRootOperations can the root dir be deleted? + * @throws IOException IO problems + */ + public static void assertDeleted(FileSystem fs, + Path file, + boolean recursive, + boolean allowRootOperations) throws IOException { + rejectRootOperation(file, allowRootOperations); + assertPathExists(fs, "about to be deleted file", file); + boolean deleted = fs.delete(file, recursive); + String dir = ls(fs, file.getParent()); + assertTrue("Delete failed on " + file + ": " + dir, deleted); + assertPathDoesNotExist(fs, "Deleted file", file); + } + + /** + * Read in "length" bytes, convert to an ascii string + * @param fs filesystem + * @param path path to read + * @param length #of bytes to read. + * @return the bytes read and converted to a string + * @throws IOException IO problems + */ + public static String readBytesToString(FileSystem fs, + Path path, + int length) throws IOException { + FSDataInputStream in = fs.open(path); + try { + byte[] buf = new byte[length]; + in.readFully(0, buf); + return toChar(buf); + } finally { + in.close(); + } + } + + /** + * Take an array of filestats and convert to a string (prefixed w/ a [01] counter + * @param stats array of stats + * @param separator separator after every entry + * @return a stringified set + */ + public static String fileStatsToString(FileStatus[] stats, String separator) { + StringBuilder buf = new StringBuilder(stats.length * 128); + for (int i = 0; i < stats.length; i++) { + buf.append(String.format("[%02d] %s", i, stats[i])).append(separator); + } + return buf.toString(); + } + + /** + * List a directory + * @param fileSystem FS + * @param path path + * @return a directory listing or failure message + * @throws IOException + */ + public static String ls(FileSystem fileSystem, Path path) throws IOException { + if (path == null) { + //surfaces when someone calls getParent() on something at the top of the path + return "/"; + } + FileStatus[] stats; + String pathtext = "ls " + path; + try { + stats = fileSystem.listStatus(path); + } catch (FileNotFoundException e) { + return pathtext + " -file not found"; + } catch (IOException e) { + return pathtext + " -failed: " + e; + } + return dumpStats(pathtext, stats); + } + + public static String dumpStats(String pathname, FileStatus[] stats) { + return pathname + fileStatsToString(stats, "\n"); + } + + /** + * Assert that a file exists and whose {@link FileStatus} entry + * declares that this is a file and not a symlink or directory. + * @param fileSystem filesystem to resolve path against + * @param filename name of the file + * @throws IOException IO problems during file operations + */ + public static void assertIsFile(FileSystem fileSystem, Path filename) throws + IOException { + assertPathExists(fileSystem, "Expected file", filename); + FileStatus status = fileSystem.getFileStatus(filename); + assertIsFile(filename, status); + } + + /** + * Assert that a file exists and whose {@link FileStatus} entry + * declares that this is a file and not a symlink or directory. + * @param filename name of the file + * @param status file status + */ + public static void assertIsFile(Path filename, FileStatus status) { + String fileInfo = filename + " " + status; + assertFalse("File claims to be a directory " + fileInfo, + status.isDirectory()); + assertFalse("File claims to be a symlink " + fileInfo, + status.isSymlink()); + } + + /** + * Create a dataset for use in the tests; all data is in the range + * base to (base+modulo-1) inclusive + * @param len length of data + * @param base base of the data + * @param modulo the modulo + * @return the newly generated dataset + */ + public static byte[] dataset(int len, int base, int modulo) { + byte[] dataset = new byte[len]; + for (int i = 0; i < len; i++) { + dataset[i] = (byte) (base + (i % modulo)); + } + return dataset; + } + + /** + * Assert that a path exists -but make no assertions as to the + * type of that entry + * + * @param fileSystem filesystem to examine + * @param message message to include in the assertion failure message + * @param path path in the filesystem + * @throws FileNotFoundException raised if the path is missing + * @throws IOException IO problems + */ + public static void assertPathExists(FileSystem fileSystem, String message, + Path path) throws IOException { + if (!fileSystem.exists(path)) { + //failure, report it + ls(fileSystem, path.getParent()); + throw new FileNotFoundException(message + ": not found " + path + + " in " + path.getParent()); + } + } + + /** + * Assert that a path does not exist + * + * @param fileSystem filesystem to examine + * @param message message to include in the assertion failure message + * @param path path in the filesystem + * @throws IOException IO problems + */ + public static void assertPathDoesNotExist(FileSystem fileSystem, + String message, + Path path) throws IOException { + try { + FileStatus status = fileSystem.getFileStatus(path); + fail(message + ": unexpectedly found " + path + " as " + status); + } catch (FileNotFoundException expected) { + //this is expected + + } + } + + /** + * Assert that a FileSystem.listStatus on a dir finds the subdir/child entry + * @param fs filesystem + * @param dir directory to scan + * @param subdir full path to look for + * @throws IOException IO probles + */ + public static void assertListStatusFinds(FileSystem fs, + Path dir, + Path subdir) throws IOException { + FileStatus[] stats = fs.listStatus(dir); + boolean found = false; + StringBuilder builder = new StringBuilder(); + for (FileStatus stat : stats) { + builder.append(stat.toString()).append('\n'); + if (stat.getPath().equals(subdir)) { + found = true; + } + } + assertTrue("Path " + subdir + + " not found in directory " + dir + ":" + builder, + found); + } + + /** + * Test for the host being an OSX machine + * @return true if the JVM thinks that is running on OSX + */ + public static boolean isOSX() { + return System.getProperty("os.name").contains("OS X"); + } + + /** + * compare content of file operations using a double byte array + * @param concat concatenated files + * @param bytes bytes + */ + public static void validateFileContent(byte[] concat, byte[][] bytes) { + int idx = 0; + boolean mismatch = false; + + for (byte[] bb : bytes) { + for (byte b : bb) { + if (b != concat[idx++]) { + mismatch = true; + break; + } + } + if (mismatch) + break; + } + assertFalse("File content of file is not as expected at offset " + idx, + mismatch); + } + + +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ftp/FTPContract.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ftp/FTPContract.java new file mode 100644 index 00000000000..1efd7fc4e95 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ftp/FTPContract.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.ftp; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.contract.AbstractBondedFSContract; +import org.junit.Assert; + +import java.net.URI; + +import static org.junit.Assert.assertNotNull; + +/** + * The contract of FTP; requires the option "test.testdir" to be set + */ +public class FTPContract extends AbstractBondedFSContract { + + public static final String CONTRACT_XML = "contract/ftp.xml"; + /** + * + */ + public static final String TEST_FS_TESTDIR = "test.ftp.testdir"; + private String fsName; + private URI fsURI; + private FileSystem fs; + + public FTPContract(Configuration conf) { + super(conf); + //insert the base features + addConfResource(CONTRACT_XML); + } + + @Override + public String getScheme() { + return "ftp"; + } + + @Override + public Path getTestPath() { + String pathString = getOption(TEST_FS_TESTDIR, null); + assertNotNull("Undefined test option " + TEST_FS_TESTDIR, pathString); + Path path = new Path(pathString); + return path; + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ftp/TestFTPContractCreate.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ftp/TestFTPContractCreate.java new file mode 100644 index 00000000000..823ab8ca035 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ftp/TestFTPContractCreate.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.ftp; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractCreateTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +public class TestFTPContractCreate extends AbstractContractCreateTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new FTPContract(conf); + } + +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ftp/TestFTPContractDelete.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ftp/TestFTPContractDelete.java new file mode 100644 index 00000000000..e749782bf13 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ftp/TestFTPContractDelete.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.ftp; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractDeleteTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +public class TestFTPContractDelete extends AbstractContractDeleteTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new FTPContract(conf); + } + +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ftp/TestFTPContractMkdir.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ftp/TestFTPContractMkdir.java new file mode 100644 index 00000000000..296298c205b --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ftp/TestFTPContractMkdir.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.ftp; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractMkdirTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * Test dir operations on a the local FS. + */ +public class TestFTPContractMkdir extends AbstractContractMkdirTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new FTPContract(conf); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ftp/TestFTPContractOpen.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ftp/TestFTPContractOpen.java new file mode 100644 index 00000000000..2df0bbc3ed4 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ftp/TestFTPContractOpen.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.ftp; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractOpenTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +public class TestFTPContractOpen extends AbstractContractOpenTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new FTPContract(conf); + } + +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ftp/TestFTPContractRename.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ftp/TestFTPContractRename.java new file mode 100644 index 00000000000..fb8718bf246 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ftp/TestFTPContractRename.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.ftp; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractRenameTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.apache.hadoop.fs.ftp.FTPFileSystem; + +import java.io.IOException; + +public class TestFTPContractRename extends AbstractContractRenameTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new FTPContract(conf); + } + + /** + * Check the exception was about cross-directory renames + * -if not, rethrow it. + * @param e exception raised + * @throws IOException + */ + private void verifyUnsupportedDirRenameException(IOException e) throws IOException { + if (!e.toString().contains(FTPFileSystem.E_SAME_DIRECTORY_ONLY)) { + throw e; + } + } + + @Override + public void testRenameDirIntoExistingDir() throws Throwable { + try { + super.testRenameDirIntoExistingDir(); + fail("Expected a failure"); + } catch (IOException e) { + verifyUnsupportedDirRenameException(e); + } + } + + @Override + public void testRenameFileNonexistentDir() throws Throwable { + try { + super.testRenameFileNonexistentDir(); + fail("Expected a failure"); + } catch (IOException e) { + verifyUnsupportedDirRenameException(e); + } + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ftp/package.html b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ftp/package.html new file mode 100644 index 00000000000..744bc81d487 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ftp/package.html @@ -0,0 +1,56 @@ + + + + + + FTP Contract Tests + + +

    FTP Contract

    + +This package contains tests that verify the FTP filesystem works close to what +a Hadoop application expects. +

    +All these tests are skipped unless a test filesystem is provided +in hadoop-common/src/test/resources/core-site.xml +
    +
+  <property>
+    <name>fs.ftp.contract.test.fs.name</name>
+    <value>ftp://ftpserver/</value>
+  </property>
+
+  <property>
+    <name>fs.ftp.contract.test.testdir</name>
+    <value>/home/testuser/test</value>
+  </property>
+
+  <property>
+    <name>fs.ftp.user.ftpserver</name>
+    <value>testuser</value>
+  </property>
+
+  <property>
+    <name>fs.ftp.password.ftpserver</name>
+    <value>remember-not-to-check-this-file-in</value>
+  </property>
+
    + + + \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/LocalFSContract.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/LocalFSContract.java new file mode 100644 index 00000000000..8053335b9f8 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/LocalFSContract.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.localfs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.apache.hadoop.fs.contract.ContractOptions; +import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.apache.hadoop.util.Shell; + +import java.io.IOException; + +/** + * The contract of the Local filesystem. + * This changes its feature set from platform for platform -the default + * set is updated during initialization. + * + * This contract contains some override points, to permit + * the raw local filesystem and other filesystems to subclass it. + */ +public class LocalFSContract extends AbstractFSContract { + + public static final String CONTRACT_XML = "contract/localfs.xml"; + public static final String SYSPROP_TEST_BUILD_DATA = "test.build.data"; + public static final String DEFAULT_TEST_BUILD_DATA_DIR = "test/build/data"; + private FileSystem fs; + + public LocalFSContract(Configuration conf) { + super(conf); + //insert the base features + addConfResource(getContractXml()); + } + + /** + * Return the contract file for this filesystem + * @return the XML + */ + protected String getContractXml() { + return CONTRACT_XML; + } + + @Override + public void init() throws IOException { + super.init(); + fs = getLocalFS(); + adjustContractToLocalEnvironment(); + } + + /** + * tweak some of the contract parameters based on the local system + * state + */ + protected void adjustContractToLocalEnvironment() { + if (Shell.WINDOWS) { + //NTFS doesn't do case sensitivity, and its permissions are ACL-based + getConf().setBoolean(getConfKey(ContractOptions.IS_CASE_SENSITIVE), false); + getConf().setBoolean(getConfKey(ContractOptions.SUPPORTS_UNIX_PERMISSIONS), false); + } else if (ContractTestUtils.isOSX()) { + //OSX HFS+ is not case sensitive + getConf().setBoolean(getConfKey(ContractOptions.IS_CASE_SENSITIVE), + false); + } + } + + /** + * Get the local filesystem. This may be overridden + * @return the filesystem + * @throws IOException + */ + protected FileSystem getLocalFS() throws IOException { + return FileSystem.getLocal(getConf()); + } + + @Override + public FileSystem getTestFileSystem() throws IOException { + return fs; + } + + @Override + public String getScheme() { + return "file"; + } + + @Override + public Path getTestPath() { + Path path = fs.makeQualified(new Path( + getTestDataDir())); + return path; + } + + /** + * Get the test data directory + * @return the directory for test data + */ + protected String getTestDataDir() { + return System.getProperty(SYSPROP_TEST_BUILD_DATA, DEFAULT_TEST_BUILD_DATA_DIR); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractAppend.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractAppend.java new file mode 100644 index 00000000000..5c0da189c72 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractAppend.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.localfs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractAppendTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +public class TestLocalFSContractAppend extends AbstractContractAppendTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new LocalFSContract(conf); + } + +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractCreate.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractCreate.java new file mode 100644 index 00000000000..f8eeb961e92 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractCreate.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.localfs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractCreateTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +public class TestLocalFSContractCreate extends AbstractContractCreateTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new LocalFSContract(conf); + } + +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractDelete.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractDelete.java new file mode 100644 index 00000000000..58484282d89 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractDelete.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.localfs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractDeleteTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +public class TestLocalFSContractDelete extends AbstractContractDeleteTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new LocalFSContract(conf); + } + +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractLoaded.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractLoaded.java new file mode 100644 index 00000000000..3b9ea4c4a15 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractLoaded.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.localfs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.apache.hadoop.fs.contract.AbstractFSContractTestBase; +import org.junit.Test; + +import java.net.URL; + +/** + * just here to make sure that the local.xml resource is actually loading + */ +public class TestLocalFSContractLoaded extends AbstractFSContractTestBase { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new LocalFSContract(conf); + } + + @Test + public void testContractWorks() throws Throwable { + String key = getContract().getConfKey(SUPPORTS_ATOMIC_RENAME); + assertNotNull("not set: " + key, getContract().getConf().get(key)); + assertTrue("not true: " + key, + getContract().isSupported(SUPPORTS_ATOMIC_RENAME, false)); + } + + @Test + public void testContractResourceOnClasspath() throws Throwable { + URL url = this.getClass() + .getClassLoader() + .getResource(LocalFSContract.CONTRACT_XML); + assertNotNull("could not find contract resource", url); + } +} \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractMkdir.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractMkdir.java new file mode 100644 index 00000000000..db42a62be10 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractMkdir.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.localfs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractMkdirTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * Test dir operations on a the local FS. + */ +public class TestLocalFSContractMkdir extends AbstractContractMkdirTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new LocalFSContract(conf); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractOpen.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractOpen.java new file mode 100644 index 00000000000..1d31fe6b96a --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractOpen.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.localfs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractOpenTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +public class TestLocalFSContractOpen extends AbstractContractOpenTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new LocalFSContract(conf); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractRename.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractRename.java new file mode 100644 index 00000000000..6b7c305fbed --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractRename.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.localfs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractRenameTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +public class TestLocalFSContractRename extends AbstractContractRenameTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new LocalFSContract(conf); + } + +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractSeek.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractSeek.java new file mode 100644 index 00000000000..007188bee9b --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/localfs/TestLocalFSContractSeek.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.localfs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractSeekTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +public class TestLocalFSContractSeek extends AbstractContractSeekTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new LocalFSContract(conf); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/RawlocalFSContract.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/RawlocalFSContract.java new file mode 100644 index 00000000000..0cd32f350d8 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/RawlocalFSContract.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.rawlocal; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.contract.localfs.LocalFSContract; + +import java.io.File; +import java.io.IOException; + +/** + * Raw local filesystem. This is the inner OS-layer FS + * before checksumming is added around it. + */ +public class RawlocalFSContract extends LocalFSContract { + public RawlocalFSContract(Configuration conf) { + super(conf); + } + + public static final String RAW_CONTRACT_XML = "contract/localfs.xml"; + + @Override + protected String getContractXml() { + return RAW_CONTRACT_XML; + } + + @Override + protected FileSystem getLocalFS() throws IOException { + return FileSystem.getLocal(getConf()).getRawFileSystem(); + } + + public File getTestDirectory() { + return new File(getTestDataDir()); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawLocalContractUnderlyingFileBehavior.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawLocalContractUnderlyingFileBehavior.java new file mode 100644 index 00000000000..2cb5414caa4 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawLocalContractUnderlyingFileBehavior.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.rawlocal; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.File; + +public class TestRawLocalContractUnderlyingFileBehavior extends Assert { + + private static File testDirectory; + + @BeforeClass + public static void before() { + RawlocalFSContract contract = + new RawlocalFSContract(new Configuration()); + testDirectory = contract.getTestDirectory(); + testDirectory.mkdirs(); + assertTrue(testDirectory.isDirectory()); + + } + + @Test + public void testDeleteEmptyPath() throws Throwable { + File nonexistent = new File(testDirectory, "testDeleteEmptyPath"); + assertFalse(nonexistent.exists()); + assertFalse("nonexistent.delete() returned true", nonexistent.delete()); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractAppend.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractAppend.java new file mode 100644 index 00000000000..cb917f4ba6a --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractAppend.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.rawlocal; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractAppendTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +public class TestRawlocalContractAppend extends AbstractContractAppendTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new RawlocalFSContract(conf); + } + +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractCreate.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractCreate.java new file mode 100644 index 00000000000..39e819bf35a --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractCreate.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.rawlocal; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractCreateTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +public class TestRawlocalContractCreate extends AbstractContractCreateTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new RawlocalFSContract(conf); + } + +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractDelete.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractDelete.java new file mode 100644 index 00000000000..bd6ef70151a --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractDelete.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.rawlocal; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractDeleteTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +public class TestRawlocalContractDelete extends AbstractContractDeleteTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new RawlocalFSContract(conf); + } + +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractMkdir.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractMkdir.java new file mode 100644 index 00000000000..68e5ebc615b --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractMkdir.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.rawlocal; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractMkdirTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * Test dir operations on a the local FS. + */ +public class TestRawlocalContractMkdir extends AbstractContractMkdirTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new RawlocalFSContract(conf); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractOpen.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractOpen.java new file mode 100644 index 00000000000..a526379ded5 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractOpen.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.rawlocal; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractOpenTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +public class TestRawlocalContractOpen extends AbstractContractOpenTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new RawlocalFSContract(conf); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractRename.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractRename.java new file mode 100644 index 00000000000..9e33b34fcb6 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractRename.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.rawlocal; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractRenameTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +public class TestRawlocalContractRename extends AbstractContractRenameTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new RawlocalFSContract(conf); + } + +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractSeek.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractSeek.java new file mode 100644 index 00000000000..b7cc0476a57 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/rawlocal/TestRawlocalContractSeek.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.rawlocal; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractSeekTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +public class TestRawlocalContractSeek extends AbstractContractSeekTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new RawlocalFSContract(conf); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/NativeS3Contract.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/NativeS3Contract.java new file mode 100644 index 00000000000..ace6444b606 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/NativeS3Contract.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.s3n; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractBondedFSContract; + +/** + * The contract of S3N: only enabled if the test bucket is provided + */ +public class NativeS3Contract extends AbstractBondedFSContract { + + public static final String CONTRACT_XML = "contract/s3n.xml"; + + + public NativeS3Contract(Configuration conf) { + super(conf); + //insert the base features + addConfResource(CONTRACT_XML); + } + + @Override + public String getScheme() { + return "s3n"; + } + +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractCreate.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractCreate.java new file mode 100644 index 00000000000..e44e2b19997 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractCreate.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.s3n; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractCreateTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.apache.hadoop.fs.contract.ContractTestUtils; + +public class TestS3NContractCreate extends AbstractContractCreateTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new NativeS3Contract(conf); + } + + @Override + public void testOverwriteEmptyDirectory() throws Throwable { + ContractTestUtils.skip( + "blobstores can't distinguish empty directories from files"); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractDelete.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractDelete.java new file mode 100644 index 00000000000..1b79d274838 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractDelete.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.s3n; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractDeleteTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +public class TestS3NContractDelete extends AbstractContractDeleteTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new NativeS3Contract(conf); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractMkdir.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractMkdir.java new file mode 100644 index 00000000000..527a31dfb67 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractMkdir.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.s3n; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractMkdirTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * Test dir operations on S3 + */ +public class TestS3NContractMkdir extends AbstractContractMkdirTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new NativeS3Contract(conf); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractOpen.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractOpen.java new file mode 100644 index 00000000000..2186f28e990 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractOpen.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.s3n; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractOpenTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +public class TestS3NContractOpen extends AbstractContractOpenTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new NativeS3Contract(conf); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRename.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRename.java new file mode 100644 index 00000000000..d673416d42c --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRename.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.s3n; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractRenameTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +public class TestS3NContractRename extends AbstractContractRenameTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new NativeS3Contract(conf); + } + +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRootDir.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRootDir.java new file mode 100644 index 00000000000..94f8483309d --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractRootDir.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.s3n; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * root dir operations against an S3 bucket + */ +public class TestS3NContractRootDir extends + AbstractContractRootDirectoryTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new NativeS3Contract(conf); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractSeek.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractSeek.java new file mode 100644 index 00000000000..6d04fff013a --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/s3n/TestS3NContractSeek.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.s3n; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractSeekTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +public class TestS3NContractSeek extends AbstractContractSeekTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new NativeS3Contract(conf); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/resources/contract/ftp.xml b/hadoop-common-project/hadoop-common/src/test/resources/contract/ftp.xml new file mode 100644 index 00000000000..9c3e380718c --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/resources/contract/ftp.xml @@ -0,0 +1,84 @@ + + + + + + + fs.contract.test.root-tests-enabled + false + + + + fs.contract.is-blobstore + false + + + + fs.contract.is-case-sensitive + true + + + + fs.contract.supports-append + false + + + + fs.contract.supports-atomic-directory-delete + true + + + + fs.contract.supports-atomic-rename + true + + + + fs.contract.supports-block-locality + false + + + + fs.contract.supports-concat + false + + + + fs.contract.supports-seek + false + + + + fs.contract.rejects-seek-past-eof + true + + + + fs.contract.supports-strict-exceptions + true + + + + fs.contract.supports-unix-permissions + false + + + \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/test/resources/contract/localfs.xml b/hadoop-common-project/hadoop-common/src/test/resources/contract/localfs.xml new file mode 100644 index 00000000000..b8857eb730f --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/resources/contract/localfs.xml @@ -0,0 +1,110 @@ + + + + + + + + fs.contract.is-case-sensitive + true + + + + + fs.contract.supports-unix-permissions + true + + + + + + fs.contract.test.root-tests-enabled + false + + + + fs.contract.test.random-seek-count + 1000 + + + + fs.contract.rename-creates-dest-dirs + true + + + + fs.contract.rename-overwrites-dest + true + + + + + + fs.contract.supports-append + false + + + + fs.contract.supports-atomic-directory-delete + true + + + + fs.contract.supports-atomic-rename + true + + + + fs.contract.supports-block-locality + false + + + + fs.contract.supports-concat + false + + + + fs.contract.supports-seek + true + + + + fs.contract.supports-seek-on-closed-file + true + + + + + fs.contract.rejects-seek-past-eof + true + + + + fs.contract.supports-strict-exceptions + false + + + \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/test/resources/contract/rawlocal.xml b/hadoop-common-project/hadoop-common/src/test/resources/contract/rawlocal.xml new file mode 100644 index 00000000000..4b26cb3195d --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/resources/contract/rawlocal.xml @@ -0,0 +1,101 @@ + + + + + + + fs.contract.test.root-tests-enabled + false + + + + fs.contract.test.random-seek-count + 1000 + + + + fs.contract.is-case-sensitive + true + + + + fs.contract.supports-append + true + + + + fs.contract.supports-atomic-directory-delete + true + + + + fs.contract.supports-atomic-rename + true + + + + fs.contract.supports-block-locality + false + + + + fs.contract.supports-concat + false + + + + fs.contract.rename-creates-dest-dirs + true + + + + fs.contract.rename-overwrites-dest + true + + + + fs.contract.supports-seek + true + + + + fs.contract.supports-seek-on-closed-file + true + + + + fs.contract.rejects-seek-past-eof + false + + + + fs.contract.supports-strict-exceptions + false + + + + fs.contract.supports-unix-permissions + true + + + \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/test/resources/contract/s3n.xml b/hadoop-common-project/hadoop-common/src/test/resources/contract/s3n.xml new file mode 100644 index 00000000000..ab46178090a --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/resources/contract/s3n.xml @@ -0,0 +1,95 @@ + + + + + + + fs.contract.test.root-tests-enabled + true + + + + fs.contract.test.random-seek-count + 10 + + + + fs.contract.is-blobstore + true + + + + fs.contract.is-case-sensitive + true + + + + fs.contract.rename-returns-false-if-source-missing + true + + + + fs.contract.supports-append + false + + + + fs.contract.supports-atomic-directory-delete + false + + + + fs.contract.supports-atomic-rename + false + + + + fs.contract.supports-block-locality + false + + + + fs.contract.supports-concat + false + + + + fs.contract.supports-seek + true + + + + fs.contract.rejects-seek-past-eof + true + + + + fs.contract.supports-strict-exceptions + true + + + + fs.contract.supports-unix-permissions + false + + + \ No newline at end of file From b6fbaba8e98d2da913cbe5474cab479cd4efd191 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Thu, 3 Jul 2014 12:09:45 +0000 Subject: [PATCH 091/112] HADOOP-9361: Strictly define FileSystem APIs - HDFS portion git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1607597 13f79535-47bb-0310-9956-ffa450edef68 --- .../apache/hadoop/hdfs/DFSInputStream.java | 5 +- .../hadoop/fs/contract/hdfs/HDFSContract.java | 96 +++++++++++++++++++ .../contract/hdfs/TestHDFSContractAppend.java | 56 +++++++++++ .../contract/hdfs/TestHDFSContractConcat.java | 50 ++++++++++ .../contract/hdfs/TestHDFSContractCreate.java | 45 +++++++++ .../contract/hdfs/TestHDFSContractDelete.java | 48 ++++++++++ .../contract/hdfs/TestHDFSContractMkdir.java | 48 ++++++++++ .../contract/hdfs/TestHDFSContractOpen.java | 48 ++++++++++ .../contract/hdfs/TestHDFSContractRename.java | 45 +++++++++ .../hdfs/TestHDFSContractRootDirectory.java | 49 ++++++++++ .../contract/hdfs/TestHDFSContractSeek.java | 49 ++++++++++ .../src/test/resources/contract/hdfs.xml | 94 ++++++++++++++++++ 12 files changed, 631 insertions(+), 2 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/HDFSContract.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractAppend.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractConcat.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractCreate.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractDelete.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractMkdir.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractOpen.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractRename.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractRootDirectory.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractSeek.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/resources/contract/hdfs.xml diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java index 1dcf373896a..dd8ec1b5c17 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs; +import java.io.EOFException; import java.io.IOException; import java.net.InetSocketAddress; import java.nio.ByteBuffer; @@ -1373,10 +1374,10 @@ public long skip(long n) throws IOException { @Override public synchronized void seek(long targetPos) throws IOException { if (targetPos > getFileLength()) { - throw new IOException("Cannot seek after EOF"); + throw new EOFException("Cannot seek after EOF"); } if (targetPos < 0) { - throw new IOException("Cannot seek to negative offset"); + throw new EOFException("Cannot seek to negative offset"); } if (closed) { throw new IOException("Stream is closed!"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/HDFSContract.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/HDFSContract.java new file mode 100644 index 00000000000..258703af5b7 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/HDFSContract.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.hdfs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.apache.hadoop.fs.contract.AbstractFSContractTestBase; +import org.apache.hadoop.fs.contract.ContractOptions; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.junit.Assert; + +import java.io.IOException; + +/** + * The contract of HDFS + * This changes its feature set from platform for platform -the default + * set is updated during initialization. + */ +public class HDFSContract extends AbstractFSContract { + + public static final String CONTRACT_HDFS_XML = "contract/hdfs.xml"; + public static final int BLOCK_SIZE = AbstractFSContractTestBase.TEST_FILE_LEN; + private static MiniDFSCluster cluster; + + public HDFSContract(Configuration conf) { + super(conf); + //insert the base features + addConfResource(CONTRACT_HDFS_XML); + } + + public static void createCluster() throws IOException { + HdfsConfiguration conf = new HdfsConfiguration(); + conf.addResource(CONTRACT_HDFS_XML); + //hack in a 256 byte block size + conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE); + + cluster = + new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); + cluster.waitClusterUp(); + } + + public static void destroyCluster() throws IOException { + if (cluster != null) { + cluster.shutdown(); + } + } + + public static MiniDFSCluster getCluster() { + return cluster; + } + + @Override + public void init() throws IOException { + super.init(); + Assert.assertTrue("contract options not loaded", + isSupported(ContractOptions.IS_CASE_SENSITIVE, false)); + } + + @Override + public FileSystem getTestFileSystem() throws IOException { + //assumes cluster is not null + Assert.assertNotNull("cluster not created", cluster); + return cluster.getFileSystem(); + } + + @Override + public String getScheme() { + return "hdfs"; + } + + @Override + public Path getTestPath() { + Path path = new Path("/test"); + return path; + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractAppend.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractAppend.java new file mode 100644 index 00000000000..c9148e10de4 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractAppend.java @@ -0,0 +1,56 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. See accompanying LICENSE file. + */ + +package org.apache.hadoop.fs.contract.hdfs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractAppendTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.FileNotFoundException; +import java.io.IOException; + +public class TestHDFSContractAppend extends AbstractContractAppendTest { + + @BeforeClass + public static void createCluster() throws IOException { + HDFSContract.createCluster(); + } + + @AfterClass + public static void teardownCluster() throws IOException { + HDFSContract.destroyCluster(); + } + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new HDFSContract(conf); + } + + @Override + public void testRenameFileBeingAppended() throws Throwable { + try { + super.testRenameFileBeingAppended(); + fail("Expected a FileNotFoundException"); + } catch (FileNotFoundException e) { + // downgrade + ContractTestUtils.downgrade("Renaming an open file" + + "still creates the old path", e); + + } + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractConcat.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractConcat.java new file mode 100644 index 00000000000..05587ce7e40 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractConcat.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.hdfs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractConcatTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; + +/** + * Test dir operations on a the local FS. + */ +public class TestHDFSContractConcat extends AbstractContractConcatTest { + + @BeforeClass + public static void createCluster() throws IOException { + HDFSContract.createCluster(); + // perform a simple operation on the cluster to verify it is up + HDFSContract.getCluster().getFileSystem().getDefaultBlockSize(); + } + + @AfterClass + public static void teardownCluster() throws IOException { + HDFSContract.destroyCluster(); + } + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new HDFSContract(conf); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractCreate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractCreate.java new file mode 100644 index 00000000000..b209bf130e2 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractCreate.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.hdfs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractCreateTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; + +public class TestHDFSContractCreate extends AbstractContractCreateTest { + + @BeforeClass + public static void createCluster() throws IOException { + HDFSContract.createCluster(); + } + + @AfterClass + public static void teardownCluster() throws IOException { + HDFSContract.destroyCluster(); + } + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new HDFSContract(conf); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractDelete.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractDelete.java new file mode 100644 index 00000000000..4dc4af05add --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractDelete.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.hdfs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractDeleteTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; + +/** + * Test dir operations on a the local FS. + */ +public class TestHDFSContractDelete extends AbstractContractDeleteTest { + + @BeforeClass + public static void createCluster() throws IOException { + HDFSContract.createCluster(); + } + + @AfterClass + public static void teardownCluster() throws IOException { + HDFSContract.destroyCluster(); + } + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new HDFSContract(conf); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractMkdir.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractMkdir.java new file mode 100644 index 00000000000..053429dec80 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractMkdir.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.hdfs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractMkdirTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; + +/** + * Test dir operations on a the local FS. + */ +public class TestHDFSContractMkdir extends AbstractContractMkdirTest { + + @BeforeClass + public static void createCluster() throws IOException { + HDFSContract.createCluster(); + } + + @AfterClass + public static void teardownCluster() throws IOException { + HDFSContract.destroyCluster(); + } + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new HDFSContract(conf); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractOpen.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractOpen.java new file mode 100644 index 00000000000..125e8eec935 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractOpen.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.hdfs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractOpenTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; + +/** + * Test dir operations on a the local FS. + */ +public class TestHDFSContractOpen extends AbstractContractOpenTest { + + @BeforeClass + public static void createCluster() throws IOException { + HDFSContract.createCluster(); + } + + @AfterClass + public static void teardownCluster() throws IOException { + HDFSContract.destroyCluster(); + } + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new HDFSContract(conf); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractRename.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractRename.java new file mode 100644 index 00000000000..706b0cf8264 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractRename.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.hdfs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractRenameTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; + +public class TestHDFSContractRename extends AbstractContractRenameTest { + + @BeforeClass + public static void createCluster() throws IOException { + HDFSContract.createCluster(); + } + + @AfterClass + public static void teardownCluster() throws IOException { + HDFSContract.destroyCluster(); + } + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new HDFSContract(conf); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractRootDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractRootDirectory.java new file mode 100644 index 00000000000..fc1851db5fb --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractRootDirectory.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.hdfs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; + +/** + * Test dir operations on a the local FS. + */ +public class TestHDFSContractRootDirectory extends + AbstractContractRootDirectoryTest { + + @BeforeClass + public static void createCluster() throws IOException { + HDFSContract.createCluster(); + } + + @AfterClass + public static void teardownCluster() throws IOException { + HDFSContract.destroyCluster(); + } + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new HDFSContract(conf); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractSeek.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractSeek.java new file mode 100644 index 00000000000..259ffce824c --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/contract/hdfs/TestHDFSContractSeek.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.contract.hdfs; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractSeekTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; + +/** + * Test dir operations on a the local FS. + */ +public class TestHDFSContractSeek extends AbstractContractSeekTest { + + @BeforeClass + public static void createCluster() throws IOException { + HDFSContract.createCluster(); + } + + @AfterClass + public static void teardownCluster() throws IOException { + HDFSContract.destroyCluster(); + } + + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new HDFSContract(conf); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/contract/hdfs.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/contract/hdfs.xml new file mode 100644 index 00000000000..51f479a66a9 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/contract/hdfs.xml @@ -0,0 +1,94 @@ + + + + + + + fs.contract.test.root-tests-enabled + true + + + + fs.file.contract.test.random-seek-count + 500 + + + + fs.contract.is-case-sensitive + true + + + + fs.contract.supports-append + true + + + + fs.contract.supports-atomic-directory-delete + true + + + + fs.contract.supports-atomic-rename + true + + + + fs.contract.supports-block-locality + true + + + + fs.contract.supports-concat + true + + + + fs.contract.supports-seek + true + + + + fs.contract.rejects-seek-past-eof + true + + + + fs.contract.supports-strict-exceptions + true + + + + fs.contract.supports-unix-permissions + true + + + + fs.contract.rename-returns-false-if-dest-exists + true + + + + fs.contract.rename-returns-false-if-source-missing + true + + + \ No newline at end of file From becc23fb65a5c2b632034b8b2c4c08832d47fd96 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Thu, 3 Jul 2014 12:10:52 +0000 Subject: [PATCH 092/112] HADOOP-9361: Strictly define FileSystem APIs - OpenStack portion git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1607599 13f79535-47bb-0310-9956-ffa450edef68 --- .../snative/StrictBufferedFSInputStream.java | 6 +- .../swift/snative/SwiftNativeFileSystem.java | 17 ++-- .../snative/SwiftNativeFileSystemStore.java | 5 +- .../swift/snative/SwiftNativeInputStream.java | 4 +- .../snative/SwiftNativeOutputStream.java | 3 +- .../fs/swift/TestSwiftFileSystemBasicOps.java | 4 +- .../fs/swift/TestSwiftFileSystemContract.java | 15 ++- .../fs/swift/TestSwiftFileSystemRename.java | 7 +- .../fs/swift/contract/SwiftContract.java | 44 +++++++++ .../contract/TestSwiftContractCreate.java | 37 ++++++++ .../contract/TestSwiftContractDelete.java | 31 ++++++ .../contract/TestSwiftContractMkdir.java | 34 +++++++ .../swift/contract/TestSwiftContractOpen.java | 42 ++++++++ .../contract/TestSwiftContractRename.java | 32 +++++++ .../contract/TestSwiftContractRootDir.java | 35 +++++++ .../swift/contract/TestSwiftContractSeek.java | 31 ++++++ .../fs/swift/hdfs2/TestV2LsOperations.java | 4 +- .../src/test/resources/contract/swift.xml | 95 +++++++++++++++++++ 18 files changed, 424 insertions(+), 22 deletions(-) create mode 100644 hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/SwiftContract.java create mode 100644 hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractCreate.java create mode 100644 hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractDelete.java create mode 100644 hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractMkdir.java create mode 100644 hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractOpen.java create mode 100644 hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractRename.java create mode 100644 hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractRootDir.java create mode 100644 hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractSeek.java create mode 100644 hadoop-tools/hadoop-openstack/src/test/resources/contract/swift.xml diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/StrictBufferedFSInputStream.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/StrictBufferedFSInputStream.java index 701f5100aaa..794219f31a4 100644 --- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/StrictBufferedFSInputStream.java +++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/StrictBufferedFSInputStream.java @@ -19,9 +19,11 @@ package org.apache.hadoop.fs.swift.snative; import org.apache.hadoop.fs.BufferedFSInputStream; +import org.apache.hadoop.fs.FSExceptionMessages; import org.apache.hadoop.fs.FSInputStream; import org.apache.hadoop.fs.swift.exceptions.SwiftConnectionClosedException; +import java.io.EOFException; import java.io.IOException; /** @@ -37,10 +39,10 @@ public StrictBufferedFSInputStream(FSInputStream in, @Override public void seek(long pos) throws IOException { if (pos < 0) { - throw new IOException("Negative position"); + throw new EOFException(FSExceptionMessages.NEGATIVE_SEEK); } if (in == null) { - throw new SwiftConnectionClosedException("Stream closed"); + throw new SwiftConnectionClosedException(FSExceptionMessages.STREAM_IS_CLOSED); } super.seek(pos); } diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystem.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystem.java index f4a4bd8a1e9..b70f7efef58 100644 --- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystem.java +++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystem.java @@ -25,14 +25,14 @@ import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.ParentNotDirectoryException; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException; -import org.apache.hadoop.fs.swift.exceptions.SwiftNotDirectoryException; import org.apache.hadoop.fs.swift.exceptions.SwiftOperationFailedException; -import org.apache.hadoop.fs.swift.exceptions.SwiftPathExistsException; import org.apache.hadoop.fs.swift.exceptions.SwiftUnsupportedFeatureException; import org.apache.hadoop.fs.swift.http.SwiftProtocolConstants; import org.apache.hadoop.fs.swift.util.DurationStats; @@ -373,7 +373,7 @@ private boolean mkdir(Path path) throws IOException { * @param directory path to query * @return true iff the directory should be created * @throws IOException IO problems - * @throws SwiftNotDirectoryException if the path references a file + * @throws ParentNotDirectoryException if the path references a file */ private boolean shouldCreate(Path directory) throws IOException { FileStatus fileStatus; @@ -388,9 +388,9 @@ private boolean shouldCreate(Path directory) throws IOException { if (!SwiftUtils.isDirectory(fileStatus)) { //if it's a file, raise an error - throw new SwiftNotDirectoryException(directory, - String.format(": can't mkdir since it exists and is not a directory: %s", - fileStatus)); + throw new ParentNotDirectoryException( + String.format("%s: can't mkdir since it exists and is not a directory: %s", + directory, fileStatus)); } else { //path exists, and it is a directory if (LOG.isDebugEnabled()) { @@ -488,7 +488,7 @@ public FSDataOutputStream create(Path file, FsPermission permission, //overwrite set -> delete the object. store.delete(absolutePath, true); } else { - throw new SwiftPathExistsException("Path exists: " + file); + throw new FileAlreadyExistsException("Path exists: " + file); } } else { // destination does not exist -trigger creation of the parent @@ -580,6 +580,9 @@ public boolean rename(Path src, Path dst) throws IOException { } catch (SwiftOperationFailedException e) { //downgrade to a failure return false; + } catch (FileAlreadyExistsException e) { + //downgrade to a failure + return false; } catch (FileNotFoundException e) { //downgrade to a failure return false; diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystemStore.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystemStore.java index e42cb442c93..b3e6b941795 100644 --- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystemStore.java +++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystemStore.java @@ -22,6 +22,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException; @@ -590,7 +591,7 @@ public void rename(Path src, Path dst) } else { //outcome #1 dest it's a file: fail if differeent if (!renamingOnToSelf) { - throw new SwiftOperationFailedException( + throw new FileAlreadyExistsException( "cannot rename a file over one that already exists"); } else { //is mv self self where self is a file. this becomes a no-op @@ -633,7 +634,7 @@ public void rename(Path src, Path dst) if (destExists && !destIsDir) { // #1 destination is a file: fail - throw new SwiftOperationFailedException( + throw new FileAlreadyExistsException( "the source is a directory, but not the destination"); } Path targetPath; diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeInputStream.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeInputStream.java index 6574762e406..3fd370227fd 100644 --- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeInputStream.java +++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeInputStream.java @@ -20,6 +20,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FSExceptionMessages; import org.apache.hadoop.fs.FSInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -298,7 +299,8 @@ private int chompBytes(long bytes) throws IOException { @Override public synchronized void seek(long targetPos) throws IOException { if (targetPos < 0) { - throw new IOException("Negative Seek offset not supported"); + throw new EOFException( + FSExceptionMessages.NEGATIVE_SEEK); } //there's some special handling of near-local data //as the seek can be omitted if it is in/adjacent diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeOutputStream.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeOutputStream.java index 4603ded264d..74710dee84c 100644 --- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeOutputStream.java +++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeOutputStream.java @@ -22,6 +22,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.swift.exceptions.SwiftConnectionClosedException; import org.apache.hadoop.fs.swift.exceptions.SwiftException; import org.apache.hadoop.fs.swift.exceptions.SwiftInternalStateException; import org.apache.hadoop.fs.swift.util.SwiftUtils; @@ -109,7 +110,7 @@ public void flush() throws IOException { */ private synchronized void verifyOpen() throws SwiftException { if (closed) { - throw new SwiftException("Output stream is closed"); + throw new SwiftConnectionClosedException(); } } diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBasicOps.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBasicOps.java index 8ad0ca49a35..c7e8b579165 100644 --- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBasicOps.java +++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBasicOps.java @@ -21,9 +21,9 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.ParentNotDirectoryException; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.swift.exceptions.SwiftBadRequestException; -import org.apache.hadoop.fs.swift.exceptions.SwiftNotDirectoryException; import org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem; import org.apache.hadoop.fs.swift.util.SwiftTestUtils; import org.junit.Test; @@ -245,7 +245,7 @@ public void testCreateDirWithFileParent() throws Throwable { writeTextFile(fs, path, "parent", true); try { fs.mkdirs(child); - } catch (SwiftNotDirectoryException expected) { + } catch (ParentNotDirectoryException expected) { LOG.debug("Expected Exception", expected); } } finally { diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemContract.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemContract.java index 42ca39a46e4..46a5f0f637e 100644 --- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemContract.java +++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemContract.java @@ -23,8 +23,8 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystemContractBaseTest; +import org.apache.hadoop.fs.ParentNotDirectoryException; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.swift.exceptions.SwiftNotDirectoryException; import org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem; import org.apache.hadoop.fs.swift.util.SwiftTestUtils; @@ -47,6 +47,14 @@ public class TestSwiftFileSystemContract private static final Log LOG = LogFactory.getLog(TestSwiftFileSystemContract.class); + /** + * Override this if the filesystem is not case sensitive + * @return true if the case detection/preservation tests should run + */ + protected boolean filesystemIsCaseSensitive() { + return false; + } + @Override protected void setUp() throws Exception { final URI uri = getFilesystemURI(); @@ -89,9 +97,8 @@ public void testMkdirsFailsForSubdirectoryOfExistingFile() throws Exception { try { fs.mkdirs(testSubDir); fail("Should throw IOException."); - } catch (SwiftNotDirectoryException e) { + } catch (ParentNotDirectoryException e) { // expected - assertEquals(filepath,e.getPath()); } //now verify that the subdir path does not exist SwiftTestUtils.assertPathDoesNotExist(fs, "subdir after mkdir", testSubDir); @@ -100,7 +107,7 @@ public void testMkdirsFailsForSubdirectoryOfExistingFile() throws Exception { try { fs.mkdirs(testDeepSubDir); fail("Should throw IOException."); - } catch (SwiftNotDirectoryException e) { + } catch (ParentNotDirectoryException e) { // expected } SwiftTestUtils.assertPathDoesNotExist(fs, "testDeepSubDir after mkdir", diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemRename.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemRename.java index 99011d834c5..f5ad155ffe3 100644 --- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemRename.java +++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemRename.java @@ -21,6 +21,7 @@ import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.swift.exceptions.SwiftOperationFailedException; import org.apache.hadoop.fs.swift.util.SwiftTestUtils; import org.junit.Test; @@ -220,7 +221,11 @@ public void testMoveDirUnderParent() throws Throwable { fs.mkdirs(testdir); Path parent = testdir.getParent(); //the outcome here is ambiguous, so is not checked - fs.rename(testdir, parent); + try { + fs.rename(testdir, parent); + } catch (SwiftOperationFailedException e) { + // allowed + } assertExists("Source directory has been deleted ", testdir); } diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/SwiftContract.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/SwiftContract.java new file mode 100644 index 00000000000..99f72b7be98 --- /dev/null +++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/SwiftContract.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.swift.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractBondedFSContract; +import org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem; + +/** + * The contract of OpenStack Swift: only enabled if the test binding data is provided + */ +public class SwiftContract extends AbstractBondedFSContract { + + public static final String CONTRACT_XML = "contract/swift.xml"; + + public SwiftContract(Configuration conf) { + super(conf); + //insert the base features + addConfResource(CONTRACT_XML); + } + + + @Override + public String getScheme() { + return SwiftNativeFileSystem.SWIFT; + } + +} diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractCreate.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractCreate.java new file mode 100644 index 00000000000..df15a0a84c3 --- /dev/null +++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractCreate.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.swift.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractCreateTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.apache.hadoop.fs.contract.ContractTestUtils; + +public class TestSwiftContractCreate extends AbstractContractCreateTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new SwiftContract(conf); + } + + @Override + public void testOverwriteEmptyDirectory() throws Throwable { + ContractTestUtils.skip("blobstores can't distinguish empty directories from files"); + } +} diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractDelete.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractDelete.java new file mode 100644 index 00000000000..65d031cd398 --- /dev/null +++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractDelete.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.swift.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractDeleteTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +public class TestSwiftContractDelete extends AbstractContractDeleteTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new SwiftContract(conf); + } +} diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractMkdir.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractMkdir.java new file mode 100644 index 00000000000..b82ba776386 --- /dev/null +++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractMkdir.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.swift.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractMkdirTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * Test dir operations on S3 + */ +public class TestSwiftContractMkdir extends AbstractContractMkdirTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new SwiftContract(conf); + } +} diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractOpen.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractOpen.java new file mode 100644 index 00000000000..0f91b6f823e --- /dev/null +++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractOpen.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.swift.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractOpenTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.apache.hadoop.fs.contract.ContractTestUtils; + +public class TestSwiftContractOpen extends AbstractContractOpenTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new SwiftContract(conf); + } + + @Override + public void testOpenReadDir() throws Throwable { + ContractTestUtils.skip("Skipping object-store quirk"); + } + + @Override + public void testOpenReadDirWithChild() throws Throwable { + ContractTestUtils.skip("Skipping object-store quirk"); + } +} diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractRename.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractRename.java new file mode 100644 index 00000000000..8f1edb9b6a3 --- /dev/null +++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractRename.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.swift.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractRenameTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +public class TestSwiftContractRename extends AbstractContractRenameTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new SwiftContract(conf); + } + +} diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractRootDir.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractRootDir.java new file mode 100644 index 00000000000..c7b766edd49 --- /dev/null +++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractRootDir.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.swift.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * root dir operations against an S3 bucket + */ +public class TestSwiftContractRootDir extends + AbstractContractRootDirectoryTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new SwiftContract(conf); + } +} diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractSeek.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractSeek.java new file mode 100644 index 00000000000..d045980e698 --- /dev/null +++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/contract/TestSwiftContractSeek.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.swift.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractSeekTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +public class TestSwiftContractSeek extends AbstractContractSeekTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new SwiftContract(conf); + } +} diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/hdfs2/TestV2LsOperations.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/hdfs2/TestV2LsOperations.java index 2adb8f4264b..833b91d57f2 100644 --- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/hdfs2/TestV2LsOperations.java +++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/hdfs2/TestV2LsOperations.java @@ -111,7 +111,7 @@ public void testListFilesRootDir() throws Throwable { @Test(timeout = SWIFT_TEST_TIMEOUT) public void testListFilesSubDir() throws Throwable { createTestSubdirs(); - Path dir = path("/test"); + Path dir = path("/test/subdir"); Path child = new Path(dir, "text.txt"); SwiftTestUtils.writeTextFile(fs, child, "text", false); assertListFilesFinds(fs, dir, child, false); @@ -120,7 +120,7 @@ public void testListFilesSubDir() throws Throwable { @Test(timeout = SWIFT_TEST_TIMEOUT) public void testListFilesRecursive() throws Throwable { createTestSubdirs(); - Path dir = path("/test"); + Path dir = path("/test/recursive"); Path child = new Path(dir, "hadoop/a/a.txt"); SwiftTestUtils.writeTextFile(fs, child, "text", false); assertListFilesFinds(fs, dir, child, true); diff --git a/hadoop-tools/hadoop-openstack/src/test/resources/contract/swift.xml b/hadoop-tools/hadoop-openstack/src/test/resources/contract/swift.xml new file mode 100644 index 00000000000..12a67e0290a --- /dev/null +++ b/hadoop-tools/hadoop-openstack/src/test/resources/contract/swift.xml @@ -0,0 +1,95 @@ + + + + + + + fs.contract.test.root-tests-enabled + true + + + + fs.contract.test.random-seek-count + 10 + + + + fs.contract.is-blobstore + true + + + + fs.contract.is-case-sensitive + true + + + + fs.contract.supports-append + false + + + + fs.contract.supports-atomic-directory-delete + false + + + + fs.contract.supports-atomic-rename + false + + + + fs.contract.supports-block-locality + false + + + + fs.contract.supports-concat + false + + + + fs.contract.supports-seek + true + + + + fs.contract.rejects-seek-past-eof + true + + + + fs.contract.supports-strict-exceptions + true + + + + fs.contract.supports-unix-permissions + false + + + + fs.contract.rename-returns-false-if-source-missing + true + + + From a10a4f72ab397bbfa0afc86b3a368265c9f1b46c Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Thu, 3 Jul 2014 12:12:52 +0000 Subject: [PATCH 093/112] HADOOP-9361: Strictly define FileSystem APIs - OpenStack portion git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1607600 13f79535-47bb-0310-9956-ffa450edef68 --- .../SwiftNotDirectoryException.java | 43 ------------------- .../exceptions/SwiftPathExistsException.java | 33 -------------- 2 files changed, 76 deletions(-) delete mode 100644 hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftNotDirectoryException.java delete mode 100644 hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftPathExistsException.java diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftNotDirectoryException.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftNotDirectoryException.java deleted file mode 100644 index 2b849dc306c..00000000000 --- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftNotDirectoryException.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.swift.exceptions; - -import org.apache.hadoop.fs.Path; - -/** - * Exception raised when an operation is meant to work on a directory, but - * the target path is not a directory - */ -public class SwiftNotDirectoryException extends SwiftException { - private final Path path; - - public SwiftNotDirectoryException(Path path) { - this(path, ""); - } - - public SwiftNotDirectoryException(Path path, - String message) { - super(path.toString() + message); - this.path = path; - } - - public Path getPath() { - return path; - } -} diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftPathExistsException.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftPathExistsException.java deleted file mode 100644 index 503b57046c1..00000000000 --- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftPathExistsException.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.swift.exceptions; - -/** - * Exception raised when trying to create a file that already exists - * and the overwrite flag is set to false. - */ -public class SwiftPathExistsException extends SwiftException { - public SwiftPathExistsException(String message) { - super(message); - } - - public SwiftPathExistsException(String message, Throwable cause) { - super(message, cause); - } -} From 6cb273c9bc992235446372e4a5853555c4602027 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Thu, 3 Jul 2014 12:13:28 +0000 Subject: [PATCH 094/112] HADOOP-9361: site and gitignore git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1607601 13f79535-47bb-0310-9956-ffa450edef68 --- .gitignore | 2 ++ hadoop-project/src/site/site.xml | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 06fd1e9bb6a..13b29ff20a3 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ target hadoop-common-project/hadoop-kms/downloads/ hadoop-hdfs-project/hadoop-hdfs/downloads hadoop-hdfs-project/hadoop-hdfs-httpfs/downloads +hadoop-common-project/hadoop-common/src/test/resources/contract-test-options.xml +hadoop-tools/hadoop-openstack/src/test/resources/contract-test-options.xml diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml index 9a22e98e119..772967feb5f 100644 --- a/hadoop-project/src/site/site.xml +++ b/hadoop-project/src/site/site.xml @@ -51,8 +51,10 @@ - + + From 8997eb2a28cd45480ee6c23a82d070c686420046 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Thu, 3 Jul 2014 12:51:08 +0000 Subject: [PATCH 095/112] HADOOP-9361: changes.txt git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1607620 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index f1cb98007ab..e7b2df543d9 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -503,6 +503,15 @@ Release 2.5.0 - UNRELEASED HADOOP-10767. Clean up unused code in Ls shell command. (cnauroth) + HADOOP-9361 Strictly define the expected behavior of filesystem APIs and + write tests to verify compliance (stevel) + + HADOOP-9651 Filesystems to throw FileAlreadyExistsException in + createFile(path, overwrite=false) when the file exists (stevel) + + HADOOP-9495 Define behaviour of Seekable.seek(), write tests, + fix all hadoop implementations for compliance + OPTIMIZATIONS BUG FIXES @@ -652,6 +661,11 @@ Release 2.5.0 - UNRELEASED HADOOP-10739. Renaming a file into a directory containing the same filename results in a confusing I/O error (chang li via jlowe) + HADOOP-10533 S3 input stream NPEs in MapReduce join (stevel) + + HADOOP-10419 BufferedFSInputStream NPEs on getPos() on a closed stream + (stevel) + BREAKDOWN OF HADOOP-10514 SUBTASKS AND RELATED JIRAS HADOOP-10520. Extended attributes definition and FileSystem APIs for From 1def6cde1fa79a37ce1df37c6410c7a6f13ccb64 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Thu, 3 Jul 2014 12:59:45 +0000 Subject: [PATCH 096/112] HADOOP-10312 changes.text updated in wrong place git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1607631 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index e7b2df543d9..beaf28d9a48 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -512,6 +512,8 @@ Release 2.5.0 - UNRELEASED HADOOP-9495 Define behaviour of Seekable.seek(), write tests, fix all hadoop implementations for compliance + HADOOP-10312 Shell.ExitCodeException to have more useful toString (stevel) + OPTIMIZATIONS BUG FIXES @@ -699,8 +701,6 @@ Release 2.5.0 - UNRELEASED HADOOP-10710. hadoop.auth cookie is not properly constructed according to RFC2109. (Juan Yu via tucu) - HADOOP-10312 Shell.ExitCodeException to have more useful toString (stevel) - Release 2.4.1 - 2014-06-23 INCOMPATIBLE CHANGES From 5cb489f9d39cdfdfcae7ec7e99f08cb86b178721 Mon Sep 17 00:00:00 2001 From: Junping Du Date: Thu, 3 Jul 2014 14:15:19 +0000 Subject: [PATCH 097/112] YARN-2242. Improve exception information on AM launch crashes. (Contributed by Li Lu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1607655 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../resourcemanager/rmapp/attempt/RMAppAttemptImpl.java | 8 ++++---- .../rmapp/attempt/TestRMAppAttemptTransitions.java | 8 ++++++++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 48af21190ac..37f90de18ff 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -26,6 +26,9 @@ Release 2.6.0 - UNRELEASED IMPROVEMENTS + YARN-2242. Improve exception information on AM launch crashes. (Li Lu + via junping_du) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 626d47efb3e..a4fe426e69f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -1273,14 +1273,14 @@ private void setAMContainerCrashedDiagnosticsAndExitStatus( this.amContainerExitStatus = status.getExitStatus(); } - private static String getAMContainerCrashedDiagnostics( + private String getAMContainerCrashedDiagnostics( RMAppAttemptContainerFinishedEvent finishEvent) { ContainerStatus status = finishEvent.getContainerStatus(); String diagnostics = "AM Container for " + finishEvent.getApplicationAttemptId() - + " exited with " + " exitCode: " + status.getExitStatus() - + " due to: " + status.getDiagnostics() + "." - + "Failing this attempt."; + + " exited with " + " exitCode: " + status.getExitStatus() + ". " + + "Check application tracking page: " + this.getTrackingUrl() + + " . Then, click on links to logs of each attempt for detailed output. "; return diagnostics; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index b2d7c0687d0..c99987d7add 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -820,6 +820,7 @@ public void testAMCrashAtAllocated() { applicationAttempt.getAppAttemptState()); verifyTokenCount(applicationAttempt.getAppAttemptId(), 1); verifyApplicationAttemptFinished(RMAppAttemptState.FAILED); + verifyAMCrashAtAllocatedDiagnosticInfo(applicationAttempt.getDiagnostics()); } @Test @@ -1237,6 +1238,13 @@ scheduler, masterService, submissionContext, new Configuration(), verifyApplicationAttemptFinished(RMAppAttemptState.FAILED); } + private void verifyAMCrashAtAllocatedDiagnosticInfo(String diagnostics) { + assertTrue("Diagnostic information does not contain application proxy URL", + diagnostics.contains(applicationAttempt.getWebProxyBase())); + assertTrue("Diagnostic information does not point the logs to the users", + diagnostics.contains("logs")); + } + private void verifyTokenCount(ApplicationAttemptId appAttemptId, int count) { verify(amRMTokenManager, times(count)).applicationMasterFinished(appAttemptId); if (UserGroupInformation.isSecurityEnabled()) { From 160c912ee63c17c1b99710b602127f2c0612c191 Mon Sep 17 00:00:00 2001 From: Uma Maheswara Rao G Date: Thu, 3 Jul 2014 16:08:44 +0000 Subject: [PATCH 098/112] HDFS-6620. Snapshot docs should specify about preserve options with cp command. Contributed by Stephen Chu. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1607685 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../src/site/xdoc/HdfsSnapshots.xml | 4 +++- .../snapshot/TestXAttrWithSnapshot.java | 23 +++++++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 3681131f834..2416359141f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -505,6 +505,9 @@ Release 2.5.0 - UNRELEASED HDFS-6610. TestShortCircuitLocalRead tests sometimes timeout on slow machines. (Charles Lamb via wang) + HDFS-6620. Snapshot docs should specify about preserve options with cp command + (Stephen Chu via umamahesh) + OPTIMIZATIONS HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml b/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml index f809e855b92..eba1d807c50 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml @@ -97,7 +97,9 @@
  • Listing the files in snapshot s0: hdfs dfs -ls /foo/.snapshot/s0
  • Copying a file from snapshot s0: - hdfs dfs -cp /foo/.snapshot/s0/bar /tmp
    • + hdfs dfs -cp -ptopax /foo/.snapshot/s0/bar /tmp +

    Note that this example uses the preserve option to preserve + timestamps, ownership, permission, ACLs and XAttrs.

    diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestXAttrWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestXAttrWithSnapshot.java index 7042fc914ca..35b687aa309 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestXAttrWithSnapshot.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestXAttrWithSnapshot.java @@ -26,6 +26,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FsShell; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.XAttrSetFlag; import org.apache.hadoop.fs.permission.FsPermission; @@ -38,6 +39,7 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.util.ToolRunner; import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; @@ -57,6 +59,7 @@ public class TestXAttrWithSnapshot { private static int pathCount = 0; private static Path path, snapshotPath; private static String snapshotName; + private final int SUCCESS = 0; // XAttrs private static final String name1 = "user.a1"; private static final byte[] value1 = { 0x31, 0x32, 0x33 }; @@ -351,6 +354,26 @@ public void testRemoveXAttrExceedsQuota() throws Exception { hdfs.removeXAttr(filePath, name1); } + /** + * Test that users can copy a snapshot while preserving its xattrs. + */ + @Test (timeout = 120000) + public void testCopySnapshotShouldPreserveXAttrs() throws Exception { + FileSystem.mkdirs(hdfs, path, FsPermission.createImmutable((short) 0700)); + hdfs.setXAttr(path, name1, value1); + hdfs.setXAttr(path, name2, value2); + SnapshotTestHelper.createSnapshot(hdfs, path, snapshotName); + Path snapshotCopy = new Path(path.toString() + "-copy"); + String[] argv = new String[] { "-cp", "-px", snapshotPath.toUri().toString(), + snapshotCopy.toUri().toString() }; + int ret = ToolRunner.run(new FsShell(conf), argv); + assertEquals("cp -px is not working on a snapshot", SUCCESS, ret); + + Map xattrs = hdfs.getXAttrs(snapshotCopy); + assertArrayEquals(value1, xattrs.get(name1)); + assertArrayEquals(value2, xattrs.get(name2)); + } + /** * Initialize the cluster, wait for it to become active, and get FileSystem * instances for our test users. From 93e23a99157c30b51752fc49748c3c210745a187 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Thu, 3 Jul 2014 17:13:59 +0000 Subject: [PATCH 099/112] HDFS-6613. Improve logging in caching classes. (wang) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1607697 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../CacheReplicationMonitor.java | 175 ++++++++---------- .../fsdataset/impl/FsDatasetCache.java | 72 +++---- .../hdfs/server/namenode/CacheManager.java | 40 ++-- .../server/namenode/TestCacheDirectives.java | 20 +- 5 files changed, 133 insertions(+), 176 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 2416359141f..78234151a3c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -262,6 +262,8 @@ Release 2.6.0 - UNRELEASED IMPROVEMENTS + HDFS-6613. Improve logging in caching classes. (wang) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java index 3d869c52191..cf5a4a6504b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java @@ -33,8 +33,6 @@ import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.UnresolvedLinkException; import org.apache.hadoop.hdfs.protocol.Block; @@ -53,8 +51,11 @@ import org.apache.hadoop.hdfs.util.ReadOnlyList; import org.apache.hadoop.util.GSet; import org.apache.hadoop.util.Time; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.common.base.Preconditions; +; /** * Scans the namesystem, scheduling blocks to be cached as appropriate. @@ -65,8 +66,8 @@ @InterfaceAudience.LimitedPrivate({"HDFS"}) public class CacheReplicationMonitor extends Thread implements Closeable { - private static final Log LOG = - LogFactory.getLog(CacheReplicationMonitor.class); + private static final Logger LOG = + LoggerFactory.getLogger(CacheReplicationMonitor.class); private final FSNamesystem namesystem; @@ -207,7 +208,7 @@ public void run() { LOG.info("Shutting down CacheReplicationMonitor."); return; } catch (Throwable t) { - LOG.fatal("Thread exiting", t); + LOG.error("Thread exiting", t); terminate(1, t); } } @@ -316,11 +317,8 @@ private void rescanCacheDirectives() { scannedDirectives++; // Skip processing this entry if it has expired if (directive.getExpiryTime() > 0 && directive.getExpiryTime() <= now) { - if (LOG.isDebugEnabled()) { - LOG.debug("Directive " + directive.getId() + ": the directive " + - "expired at " + directive.getExpiryTime() + " (now = " + - now + ")"); - } + LOG.debug("Directive {}: the directive expired at {} (now = {})", + directive.getId(), directive.getExpiryTime(), now); continue; } String path = directive.getPath(); @@ -329,17 +327,14 @@ private void rescanCacheDirectives() { node = fsDir.getINode(path); } catch (UnresolvedLinkException e) { // We don't cache through symlinks - if (LOG.isDebugEnabled()) { - LOG.debug("Directive " + directive.getId() + - ": got UnresolvedLinkException while resolving path " + path); - } + LOG.debug("Directive {}: got UnresolvedLinkException while resolving " + + "path {}", directive.getId(), path + ); continue; } if (node == null) { - if (LOG.isDebugEnabled()) { - LOG.debug("Directive " + directive.getId() + - ": No inode found at " + path); - } + LOG.debug("Directive {}: No inode found at {}", directive.getId(), + path); } else if (node.isDirectory()) { INodeDirectory dir = node.asDirectory(); ReadOnlyList children = dir @@ -352,10 +347,8 @@ private void rescanCacheDirectives() { } else if (node.isFile()) { rescanFile(directive, node.asFile()); } else { - if (LOG.isDebugEnabled()) { - LOG.debug("Directive " + directive.getId() + - ": ignoring non-directive, non-file inode " + node); - } + LOG.debug("Directive {}: ignoring non-directive, non-file inode {} ", + directive.getId(), node); } } } @@ -381,15 +374,13 @@ private void rescanFile(CacheDirective directive, INodeFile file) { // do not cache this file. CachePool pool = directive.getPool(); if (pool.getBytesNeeded() > pool.getLimit()) { - if (LOG.isDebugEnabled()) { - LOG.debug(String.format("Directive %d: not scanning file %s because " + - "bytesNeeded for pool %s is %d, but the pool's limit is %d", - directive.getId(), - file.getFullPathName(), - pool.getPoolName(), - pool.getBytesNeeded(), - pool.getLimit())); - } + LOG.debug("Directive {}: not scanning file {} because " + + "bytesNeeded for pool {} is {}, but the pool's limit is {}", + directive.getId(), + file.getFullPathName(), + pool.getPoolName(), + pool.getBytesNeeded(), + pool.getLimit()); return; } @@ -397,11 +388,10 @@ private void rescanFile(CacheDirective directive, INodeFile file) { for (BlockInfo blockInfo : blockInfos) { if (!blockInfo.getBlockUCState().equals(BlockUCState.COMPLETE)) { // We don't try to cache blocks that are under construction. - if (LOG.isTraceEnabled()) { - LOG.trace("Directive " + directive.getId() + ": can't cache " + - "block " + blockInfo + " because it is in state " + - blockInfo.getBlockUCState() + ", not COMPLETE."); - } + LOG.trace("Directive {}: can't cache block {} because it is in state " + + "{}, not COMPLETE.", directive.getId(), blockInfo, + blockInfo.getBlockUCState() + ); continue; } Block block = new Block(blockInfo.getBlockId()); @@ -415,7 +405,7 @@ private void rescanFile(CacheDirective directive, INodeFile file) { // Update bytesUsed using the current replication levels. // Assumptions: we assume that all the blocks are the same length // on each datanode. We can assume this because we're only caching - // blocks in state COMMITTED. + // blocks in state COMPLETE. // Note that if two directives are caching the same block(s), they will // both get them added to their bytesCached. List cachedOn = @@ -441,21 +431,16 @@ private void rescanFile(CacheDirective directive, INodeFile file) { ocblock.setReplicationAndMark(directive.getReplication(), mark); } } - if (LOG.isTraceEnabled()) { - LOG.trace("Directive " + directive.getId() + ": setting replication " + - "for block " + blockInfo + " to " + ocblock.getReplication()); - } + LOG.trace("Directive {}: setting replication for block {} to {}", + directive.getId(), blockInfo, ocblock.getReplication()); } // Increment the "cached" statistics directive.addBytesCached(cachedTotal); if (cachedTotal == neededTotal) { directive.addFilesCached(1); } - if (LOG.isDebugEnabled()) { - LOG.debug("Directive " + directive.getId() + ": caching " + - file.getFullPathName() + ": " + cachedTotal + "/" + neededTotal + - " bytes"); - } + LOG.debug("Directive {}: caching {}: {}/{} bytes", directive.getId(), + file.getFullPathName(), cachedTotal, neededTotal); } private String findReasonForNotCaching(CachedBlock cblock, @@ -512,11 +497,9 @@ private void rescanCachedBlockMap() { iter.hasNext(); ) { DatanodeDescriptor datanode = iter.next(); if (!cblock.isInList(datanode.getCached())) { - if (LOG.isTraceEnabled()) { - LOG.trace("Block " + cblock.getBlockId() + ": removing from " + - "PENDING_UNCACHED for node " + datanode.getDatanodeUuid() + - "because the DataNode uncached it."); - } + LOG.trace("Block {}: removing from PENDING_UNCACHED for node {} " + + "because the DataNode uncached it.", cblock.getBlockId(), + datanode.getDatanodeUuid()); datanode.getPendingUncached().remove(cblock); iter.remove(); } @@ -526,10 +509,8 @@ private void rescanCachedBlockMap() { String reason = findReasonForNotCaching(cblock, blockInfo); int neededCached = 0; if (reason != null) { - if (LOG.isTraceEnabled()) { - LOG.trace("Block " + cblock.getBlockId() + ": can't cache " + - "block because it is " + reason); - } + LOG.trace("Block {}: can't cache block because it is {}", + cblock.getBlockId(), reason); } else { neededCached = cblock.getReplication(); } @@ -541,12 +522,12 @@ private void rescanCachedBlockMap() { DatanodeDescriptor datanode = iter.next(); datanode.getPendingCached().remove(cblock); iter.remove(); - if (LOG.isTraceEnabled()) { - LOG.trace("Block " + cblock.getBlockId() + ": removing from " + - "PENDING_CACHED for node " + datanode.getDatanodeUuid() + - "because we already have " + numCached + " cached " + - "replicas and we only need " + neededCached); - } + LOG.trace("Block {}: removing from PENDING_CACHED for node {}" + + "because we already have {} cached replicas and we only" + + " need {}", + cblock.getBlockId(), datanode.getDatanodeUuid(), numCached, + neededCached + ); } } if (numCached < neededCached) { @@ -556,12 +537,11 @@ private void rescanCachedBlockMap() { DatanodeDescriptor datanode = iter.next(); datanode.getPendingUncached().remove(cblock); iter.remove(); - if (LOG.isTraceEnabled()) { - LOG.trace("Block " + cblock.getBlockId() + ": removing from " + - "PENDING_UNCACHED for node " + datanode.getDatanodeUuid() + - "because we only have " + numCached + " cached replicas " + - "and we need " + neededCached); - } + LOG.trace("Block {}: removing from PENDING_UNCACHED for node {} " + + "because we only have {} cached replicas and we need " + + "{}", cblock.getBlockId(), datanode.getDatanodeUuid(), + numCached, neededCached + ); } } int neededUncached = numCached - @@ -581,11 +561,10 @@ private void rescanCachedBlockMap() { pendingUncached.isEmpty() && pendingCached.isEmpty()) { // we have nothing more to do with this block. - if (LOG.isTraceEnabled()) { - LOG.trace("Block " + cblock.getBlockId() + ": removing from " + - "cachedBlocks, since neededCached == 0, and " + - "pendingUncached and pendingCached are empty."); - } + LOG.trace("Block {}: removing from cachedBlocks, since neededCached " + + "== 0, and pendingUncached and pendingCached are empty.", + cblock.getBlockId() + ); cbIter.remove(); } } @@ -643,18 +622,14 @@ private void addNewPendingCached(final int neededCached, BlockInfo blockInfo = blockManager. getStoredBlock(new Block(cachedBlock.getBlockId())); if (blockInfo == null) { - if (LOG.isDebugEnabled()) { - LOG.debug("Block " + cachedBlock.getBlockId() + ": can't add new " + - "cached replicas, because there is no record of this block " + - "on the NameNode."); - } + LOG.debug("Block {}: can't add new cached replicas," + + " because there is no record of this block " + + "on the NameNode.", cachedBlock.getBlockId()); return; } if (!blockInfo.isComplete()) { - if (LOG.isDebugEnabled()) { - LOG.debug("Block " + cachedBlock.getBlockId() + ": can't cache this " + - "block, because it is not yet complete."); - } + LOG.debug("Block {}: can't cache this block, because it is not yet" + + " complete.", cachedBlock.getBlockId()); return; } // Filter the list of replicas to only the valid targets @@ -678,7 +653,7 @@ private void addNewPendingCached(final int neededCached, if (pendingCached.contains(datanode) || cached.contains(datanode)) { continue; } - long pendingCapacity = datanode.getCacheRemaining(); + long pendingBytes = 0; // Subtract pending cached blocks from effective capacity Iterator it = datanode.getPendingCached().iterator(); while (it.hasNext()) { @@ -686,7 +661,7 @@ private void addNewPendingCached(final int neededCached, BlockInfo info = blockManager.getStoredBlock(new Block(cBlock.getBlockId())); if (info != null) { - pendingCapacity -= info.getNumBytes(); + pendingBytes -= info.getNumBytes(); } } it = datanode.getPendingUncached().iterator(); @@ -696,17 +671,17 @@ private void addNewPendingCached(final int neededCached, BlockInfo info = blockManager.getStoredBlock(new Block(cBlock.getBlockId())); if (info != null) { - pendingCapacity += info.getNumBytes(); + pendingBytes += info.getNumBytes(); } } + long pendingCapacity = pendingBytes + datanode.getCacheRemaining(); if (pendingCapacity < blockInfo.getNumBytes()) { - if (LOG.isTraceEnabled()) { - LOG.trace("Block " + blockInfo.getBlockId() + ": DataNode " + - datanode.getDatanodeUuid() + " is not a valid possibility " + - "because the block has size " + blockInfo.getNumBytes() + ", but " + - "the DataNode only has " + datanode.getCacheRemaining() + " " + - "bytes of cache remaining."); - } + LOG.trace("Block {}: DataNode {} is not a valid possibility " + + "because the block has size {}, but the DataNode only has {}" + + "bytes of cache remaining ({} pending bytes, {} already cached.", + blockInfo.getBlockId(), datanode.getDatanodeUuid(), + blockInfo.getNumBytes(), pendingCapacity, pendingBytes, + datanode.getCacheRemaining()); outOfCapacity++; continue; } @@ -715,22 +690,20 @@ private void addNewPendingCached(final int neededCached, List chosen = chooseDatanodesForCaching(possibilities, neededCached, blockManager.getDatanodeManager().getStaleInterval()); for (DatanodeDescriptor datanode : chosen) { - if (LOG.isTraceEnabled()) { - LOG.trace("Block " + blockInfo.getBlockId() + ": added to " + - "PENDING_CACHED on DataNode " + datanode.getDatanodeUuid()); - } + LOG.trace("Block {}: added to PENDING_CACHED on DataNode {}", + blockInfo.getBlockId(), datanode.getDatanodeUuid()); pendingCached.add(datanode); boolean added = datanode.getPendingCached().add(cachedBlock); assert added; } // We were unable to satisfy the requested replication factor if (neededCached > chosen.size()) { - if (LOG.isDebugEnabled()) { - LOG.debug("Block " + blockInfo.getBlockId() + ": we only have " + - (cachedBlock.getReplication() - neededCached + chosen.size()) + - " of " + cachedBlock.getReplication() + " cached replicas. " + - outOfCapacity + " DataNodes have insufficient cache capacity."); - } + LOG.debug("Block {}: we only have {} of {} cached replicas." + + " {} DataNodes have insufficient cache capacity.", + blockInfo.getBlockId(), + (cachedBlock.getReplication() - neededCached + chosen.size()), + cachedBlock.getReplication(), outOfCapacity + ); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java index f1c60a3069e..060aed481a5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java @@ -37,8 +37,6 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.io.IOUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.ChecksumException; @@ -47,6 +45,8 @@ import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.io.nativeio.NativeIO; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Manages caching for an FsDatasetImpl by using the mmap(2) and mlock(2) @@ -101,7 +101,8 @@ public boolean shouldAdvertise() { } } - private static final Log LOG = LogFactory.getLog(FsDatasetCache.class); + private static final Logger LOG = LoggerFactory.getLogger(FsDatasetCache + .class); /** * Stores MappableBlock objects and the states they're in. @@ -245,21 +246,17 @@ synchronized void cacheBlock(long blockId, String bpid, ExtendedBlockId key = new ExtendedBlockId(blockId, bpid); Value prevValue = mappableBlockMap.get(key); if (prevValue != null) { - if (LOG.isDebugEnabled()) { - LOG.debug("Block with id " + blockId + ", pool " + bpid + - " already exists in the FsDatasetCache with state " + - prevValue.state); - } + LOG.debug("Block with id {}, pool {} already exists in the " + + "FsDatasetCache with state {}", blockId, bpid, prevValue.state + ); numBlocksFailedToCache.incrementAndGet(); return; } mappableBlockMap.put(key, new Value(null, State.CACHING)); volumeExecutor.execute( new CachingTask(key, blockFileName, length, genstamp)); - if (LOG.isDebugEnabled()) { - LOG.debug("Initiating caching for Block with id " + blockId + - ", pool " + bpid); - } + LOG.debug("Initiating caching for Block with id {}, pool {}", blockId, + bpid); } synchronized void uncacheBlock(String bpid, long blockId) { @@ -270,44 +267,34 @@ synchronized void uncacheBlock(String bpid, long blockId) { processBlockMunlockRequest(key)) { // TODO: we probably want to forcibly uncache the block (and close the // shm) after a certain timeout has elapsed. - if (LOG.isDebugEnabled()) { - LOG.debug(key + " is anchored, and can't be uncached now."); - } + LOG.debug("{} is anchored, and can't be uncached now.", key); return; } if (prevValue == null) { - if (LOG.isDebugEnabled()) { - LOG.debug("Block with id " + blockId + ", pool " + bpid + " " + - "does not need to be uncached, because it is not currently " + - "in the mappableBlockMap."); - } + LOG.debug("Block with id {}, pool {} does not need to be uncached, " + + "because it is not currently in the mappableBlockMap.", blockId, + bpid); numBlocksFailedToUncache.incrementAndGet(); return; } switch (prevValue.state) { case CACHING: - if (LOG.isDebugEnabled()) { - LOG.debug("Cancelling caching for block with id " + blockId + - ", pool " + bpid + "."); - } + LOG.debug("Cancelling caching for block with id {}, pool {}.", blockId, + bpid); mappableBlockMap.put(key, new Value(prevValue.mappableBlock, State.CACHING_CANCELLED)); break; case CACHED: - if (LOG.isDebugEnabled()) { - LOG.debug("Block with id " + blockId + ", pool " + bpid + " " + - "has been scheduled for uncaching."); - } + LOG.debug( + "Block with id {}, pool {} has been scheduled for uncaching" + ".", + blockId, bpid); mappableBlockMap.put(key, new Value(prevValue.mappableBlock, State.UNCACHING)); uncachingExecutor.execute(new UncachingTask(key)); break; default: - if (LOG.isDebugEnabled()) { - LOG.debug("Block with id " + blockId + ", pool " + bpid + " " + - "does not need to be uncached, because it is " + - "in state " + prevValue.state + "."); - } + LOG.debug("Block with id {}, pool {} does not need to be uncached, " + + "because it is in state {}.", blockId, bpid, prevValue.state); numBlocksFailedToUncache.incrementAndGet(); break; } @@ -386,10 +373,8 @@ public void run() { } mappableBlockMap.put(key, new Value(mappableBlock, State.CACHED)); } - if (LOG.isDebugEnabled()) { - LOG.debug("Successfully cached " + key + ". We are now caching " + - newUsedBytes + " bytes in total."); - } + LOG.debug("Successfully cached {}. We are now caching {} bytes in" + + " total.", key, newUsedBytes); dataset.datanode.getShortCircuitRegistry().processBlockMlockEvent(key); numBlocksCached.addAndGet(1); dataset.datanode.getMetrics().incrBlocksCached(1); @@ -399,12 +384,10 @@ public void run() { IOUtils.closeQuietly(metaIn); if (!success) { if (reservedBytes) { - newUsedBytes = usedBytesCount.release(length); - } - if (LOG.isDebugEnabled()) { - LOG.debug("Caching of " + key + " was aborted. We are now " + - "caching only " + newUsedBytes + " + bytes in total."); + usedBytesCount.release(length); } + LOG.debug("Caching of {} was aborted. We are now caching only {} " + + "bytes in total.", key, usedBytesCount.get()); if (mappableBlock != null) { mappableBlock.close(); } @@ -444,10 +427,7 @@ public void run() { usedBytesCount.release(value.mappableBlock.getLength()); numBlocksCached.addAndGet(-1); dataset.datanode.getMetrics().incrBlocksUncached(1); - if (LOG.isDebugEnabled()) { - LOG.debug("Uncaching of " + key + " completed. " + - "usedBytes = " + newUsedBytes); - } + LOG.debug("Uncaching of {} completed. usedBytes = {}", key, newUsedBytes); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java index 1d74cf3af4b..e5270adc48a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java @@ -43,8 +43,6 @@ import java.util.concurrent.locks.ReentrantLock; import org.apache.commons.io.IOUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries; @@ -85,6 +83,8 @@ import org.apache.hadoop.util.GSet; import org.apache.hadoop.util.LightWeightGSet; import org.apache.hadoop.util.Time; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; @@ -99,7 +99,7 @@ */ @InterfaceAudience.LimitedPrivate({"HDFS"}) public final class CacheManager { - public static final Log LOG = LogFactory.getLog(CacheManager.class); + public static final Logger LOG = LoggerFactory.getLogger(CacheManager.class); private static final float MIN_CACHED_BLOCKS_PERCENT = 0.001f; @@ -205,8 +205,8 @@ public PersistState(CacheManagerSection section, DFS_NAMENODE_PATH_BASED_CACHE_BLOCK_MAP_ALLOCATION_PERCENT, DFS_NAMENODE_PATH_BASED_CACHE_BLOCK_MAP_ALLOCATION_PERCENT_DEFAULT); if (cachedBlocksPercent < MIN_CACHED_BLOCKS_PERCENT) { - LOG.info("Using minimum value " + MIN_CACHED_BLOCKS_PERCENT + - " for " + DFS_NAMENODE_PATH_BASED_CACHE_BLOCK_MAP_ALLOCATION_PERCENT); + LOG.info("Using minimum value {} for {}", MIN_CACHED_BLOCKS_PERCENT, + DFS_NAMENODE_PATH_BASED_CACHE_BLOCK_MAP_ALLOCATION_PERCENT); cachedBlocksPercent = MIN_CACHED_BLOCKS_PERCENT; } this.cachedBlocks = new LightWeightGSet( @@ -346,10 +346,8 @@ private static short validateReplication(CacheDirectiveInfo directive, */ private static long validateExpiryTime(CacheDirectiveInfo info, long maxRelativeExpiryTime) throws InvalidRequestException { - if (LOG.isTraceEnabled()) { - LOG.trace("Validating directive " + info - + " pool maxRelativeExpiryTime " + maxRelativeExpiryTime); - } + LOG.trace("Validating directive {} pool maxRelativeExpiryTime {}", info, + maxRelativeExpiryTime); final long now = new Date().getTime(); final long maxAbsoluteExpiryTime = now + maxRelativeExpiryTime; if (info == null || info.getExpiration() == null) { @@ -539,7 +537,7 @@ public CacheDirectiveInfo addDirective( LOG.warn("addDirective of " + info + " failed: ", e); throw e; } - LOG.info("addDirective of " + info + " successful."); + LOG.info("addDirective of {} successful.", info); return directive.toInfo(); } @@ -641,8 +639,7 @@ public void modifyDirective(CacheDirectiveInfo info, LOG.warn("modifyDirective of " + idString + " failed: ", e); throw e; } - LOG.info("modifyDirective of " + idString + " successfully applied " + - info+ "."); + LOG.info("modifyDirective of {} successfully applied {}.", idString, info); } private void removeInternal(CacheDirective directive) @@ -779,7 +776,7 @@ public CachePoolInfo addCachePool(CachePoolInfo info) LOG.info("addCachePool of " + info + " failed: ", e); throw e; } - LOG.info("addCachePool of " + info + " successful."); + LOG.info("addCachePool of {} successful.", info); return pool.getInfo(true); } @@ -842,8 +839,8 @@ public void modifyCachePool(CachePoolInfo info) LOG.info("modifyCachePool of " + info + " failed: ", e); throw e; } - LOG.info("modifyCachePool of " + info.getPoolName() + " successful; " - + bld.toString()); + LOG.info("modifyCachePool of {} successful; {}", info.getPoolName(), + bld.toString()); } /** @@ -935,11 +932,9 @@ public final void processCacheReport(final DatanodeID datanodeID, if (metrics != null) { metrics.addCacheBlockReport((int) (endTime - startTime)); } - if (LOG.isDebugEnabled()) { - LOG.debug("Processed cache report from " - + datanodeID + ", blocks: " + blockIds.size() - + ", processing time: " + (endTime - startTime) + " msecs"); - } + LOG.debug("Processed cache report from {}, blocks: {}, " + + "processing time: {} msecs", datanodeID, blockIds.size(), + (endTime - startTime)); } private void processCacheReportImpl(final DatanodeDescriptor datanode, @@ -950,6 +945,8 @@ private void processCacheReportImpl(final DatanodeDescriptor datanode, CachedBlocksList pendingCachedList = datanode.getPendingCached(); for (Iterator iter = blockIds.iterator(); iter.hasNext(); ) { long blockId = iter.next(); + LOG.trace("Cache report from datanode {} has block {}", datanode, + blockId); CachedBlock cachedBlock = new CachedBlock(blockId, (short)0, false); CachedBlock prevCachedBlock = cachedBlocks.get(cachedBlock); @@ -959,15 +956,18 @@ private void processCacheReportImpl(final DatanodeDescriptor datanode, cachedBlock = prevCachedBlock; } else { cachedBlocks.put(cachedBlock); + LOG.trace("Added block {} to cachedBlocks", cachedBlock); } // Add the block to the datanode's implicit cached block list // if it's not already there. Similarly, remove it from the pending // cached block list if it exists there. if (!cachedBlock.isPresent(cachedList)) { cachedList.add(cachedBlock); + LOG.trace("Added block {} to CACHED list.", cachedBlock); } if (cachedBlock.isPresent(pendingCachedList)) { pendingCachedList.remove(cachedBlock); + LOG.trace("Removed block {} from PENDING_CACHED list.", cachedBlock); } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java index bb8ef969168..d54b90e6631 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java @@ -34,6 +34,7 @@ import java.io.IOException; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; import java.util.EnumSet; import java.util.Iterator; @@ -682,6 +683,12 @@ public Boolean get() { } finally { namesystem.readUnlock(); } + + LOG.info(logString + " cached blocks: have " + numCachedBlocks + + " / " + expectedCachedBlocks + ". " + + "cached replicas: have " + numCachedReplicas + + " / " + expectedCachedReplicas); + if (expectedCachedBlocks == -1 || numCachedBlocks == expectedCachedBlocks) { if (expectedCachedReplicas == -1 || @@ -689,10 +696,6 @@ public Boolean get() { return true; } } - LOG.info(logString + " cached blocks: have " + numCachedBlocks + - " / " + expectedCachedBlocks + ". " + - "cached replicas: have " + numCachedReplicas + - " / " + expectedCachedReplicas); return false; } }, 500, 60000); @@ -1415,7 +1418,10 @@ private void checkPendingCachedEmpty(MiniDFSCluster cluster) for (DataNode dn : cluster.getDataNodes()) { DatanodeDescriptor descriptor = datanodeManager.getDatanode(dn.getDatanodeId()); - Assert.assertTrue(descriptor.getPendingCached().isEmpty()); + Assert.assertTrue("Pending cached list of " + descriptor + + " is not empty, " + + Arrays.toString(descriptor.getPendingCached().toArray()), + descriptor.getPendingCached().isEmpty()); } } finally { cluster.getNamesystem().readUnlock(); @@ -1430,10 +1436,6 @@ public void testExceedsCapacity() throws Exception { int numCachedReplicas = (int) ((CACHE_CAPACITY*NUM_DATANODES)/BLOCK_SIZE); DFSTestUtil.createFile(dfs, fileName, fileLen, (short) NUM_DATANODES, 0xFADED); - // Set up a log appender watcher - final LogVerificationAppender appender = new LogVerificationAppender(); - final Logger logger = Logger.getRootLogger(); - logger.addAppender(appender); dfs.addCachePool(new CachePoolInfo("pool")); dfs.addCacheDirective(new CacheDirectiveInfo.Builder().setPool("pool") .setPath(fileName).setReplication((short) 1).build()); From f881f0e92317d6badb71f80fd9ea7649644e9419 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Thu, 3 Jul 2014 20:02:14 +0000 Subject: [PATCH 100/112] HDFS-6493. Change dfs.namenode.startup.delay.block.deletion to second instead of millisecond. Contributed by Juan Yu. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1607732 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 ++-- .../hdfs/server/blockmanagement/BlockManager.java | 4 ++-- .../server/blockmanagement/InvalidateBlocks.java | 5 +++-- .../hadoop-hdfs/src/main/resources/hdfs-default.xml | 12 ++++++++++++ .../blockmanagement/TestPendingInvalidateBlock.java | 2 +- 6 files changed, 23 insertions(+), 7 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 78234151a3c..75223992e65 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -510,6 +510,9 @@ Release 2.5.0 - UNRELEASED HDFS-6620. Snapshot docs should specify about preserve options with cp command (Stephen Chu via umamahesh) + HDFS-6493. Change dfs.namenode.startup.delay.block.deletion to second + instead of millisecond. (Juan Yu via wang) + OPTIMIZATIONS HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 2d825effb85..a43106943bb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -252,8 +252,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final long DFS_NAMENODE_PATH_BASED_CACHE_REFRESH_INTERVAL_MS_DEFAULT = 30000L; /** Pending period of block deletion since NameNode startup */ - public static final String DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_MS_KEY = "dfs.namenode.startup.delay.block.deletion.ms"; - public static final long DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_MS_DEFAULT = 0L; + public static final String DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY = "dfs.namenode.startup.delay.block.deletion.sec"; + public static final long DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_DEFAULT = 0L; // Whether to enable datanode's stale state detection and usage for reads public static final String DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_KEY = "dfs.namenode.avoid.read.stale.datanode"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 6a20a406211..b200f4daf40 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -263,8 +263,8 @@ public BlockManager(final Namesystem namesystem, final FSClusterStats stats, heartbeatManager = datanodeManager.getHeartbeatManager(); final long pendingPeriod = conf.getLong( - DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_MS_KEY, - DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_MS_DEFAULT); + DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY, + DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_DEFAULT) * 1000L; invalidateBlocks = new InvalidateBlocks( datanodeManager.blockInvalidateLimit, pendingPeriod); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java index 8aca559f899..66649bb4e37 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.util.LightWeightHashSet; import org.apache.hadoop.util.Time; +import org.apache.hadoop.hdfs.DFSUtil; import com.google.common.annotations.VisibleForTesting; @@ -67,8 +68,8 @@ class InvalidateBlocks { } private void printBlockDeletionTime(final Log log) { - log.info(DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_MS_KEY - + " is set to " + pendingPeriodInMs + " ms."); + log.info(DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY + + " is set to " + DFSUtil.durationToString(pendingPeriodInMs)); SimpleDateFormat sdf = new SimpleDateFormat("yyyy MMM dd HH:mm:ss"); Calendar calendar = new GregorianCalendar(); calendar.add(Calendar.SECOND, (int) (this.pendingPeriodInMs / 1000)); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index c7b0bae494d..1a1f22aaf17 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -1996,4 +1996,16 @@ + + dfs.namenode.startup.delay.block.deletion.sec + 0 + The delay in seconds at which we will pause the blocks deletion + after Namenode startup. By default it's disabled. + In the case a directory has large number of directories and files are + deleted, suggested delay is one hour to give the administrator enough time + to notice large number of pending deletion blocks and take corrective + action. + + + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java index 35e27b0878f..fd3b815b492 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java @@ -55,7 +55,7 @@ public void setUp() throws Exception { conf = new Configuration(); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCKSIZE); // block deletion pending period - conf.setLong(DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_MS_KEY, 1000 * 5); + conf.setLong(DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY, 5L); // set the block report interval to 2s conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 2000); conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1); From 151c5deaf15f7c13efba9b9141089804196fae92 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Thu, 3 Jul 2014 20:10:54 +0000 Subject: [PATCH 101/112] HDFS-6511. BlockManager#computeInvalidateWork() could do nothing. Contributed by Juan Yu. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1607735 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../hdfs/server/blockmanagement/BlockManager.java | 10 ++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 75223992e65..0f04bc18795 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -264,6 +264,8 @@ Release 2.6.0 - UNRELEASED HDFS-6613. Improve logging in caching classes. (wang) + HDFS-6511. BlockManager#computeInvalidateWork() could do nothing. (Juan Yu via wang) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index b200f4daf40..dbecf5a6319 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -1226,8 +1226,14 @@ int computeInvalidateWork(int nodesToProcess) { nodesToProcess = Math.min(nodes.size(), nodesToProcess); int blockCnt = 0; - for(int nodeCnt = 0; nodeCnt < nodesToProcess; nodeCnt++ ) { - blockCnt += invalidateWorkForOneNode(nodes.get(nodeCnt)); + for (DatanodeInfo dnInfo : nodes) { + int blocks = invalidateWorkForOneNode(dnInfo); + if (blocks > 0) { + blockCnt += blocks; + if (--nodesToProcess == 0) { + break; + } + } } return blockCnt; } From 4074843fea2df972ef62bae5b05613538c63fc89 Mon Sep 17 00:00:00 2001 From: Junping Du Date: Fri, 4 Jul 2014 11:18:05 +0000 Subject: [PATCH 102/112] YARN-2251. Avoid negative elapsed time in JHS/MRAM web UI and services (Contributed by Zhijie Shen) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1607833 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../org/apache/hadoop/yarn/util/Times.java | 26 +++++++++++++++++-- .../apache/hadoop/yarn/util/TestTimes.java | 11 ++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 37f90de18ff..921ba31d68b 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -33,6 +33,9 @@ Release 2.6.0 - UNRELEASED BUG FIXES + YARN-2251. Avoid negative elapsed time in JHS/MRAM web UI and services. + (Zhijie Shen via junping_du) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Times.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Times.java index b36edecfabe..92cc72ae1d7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Times.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Times.java @@ -21,10 +21,14 @@ import java.text.SimpleDateFormat; import java.util.Date; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @Private public class Times { + private static final Log LOG = LogFactory.getLog(Times.class); + static final ThreadLocal dateFormat = new ThreadLocal() { @Override protected SimpleDateFormat initialValue() { @@ -36,12 +40,30 @@ public static long elapsed(long started, long finished) { return Times.elapsed(started, finished, true); } + // A valid elapsed is supposed to be non-negative. If finished/current time + // is ahead of the started time, return -1 to indicate invalid elapsed time, + // and record a warning log. public static long elapsed(long started, long finished, boolean isRunning) { if (finished > 0 && started > 0) { - return finished - started; + long elapsed = finished - started; + if (elapsed >= 0) { + return elapsed; + } else { + LOG.warn("Finished time " + finished + + " is ahead of started time " + started); + return -1; + } } if (isRunning) { - return started > 0 ? System.currentTimeMillis() - started : 0; + long current = System.currentTimeMillis(); + long elapsed = started > 0 ? current - started : 0; + if (elapsed >= 0) { + return elapsed; + } else { + LOG.warn("Current time " + current + + " is ahead of started time " + started); + return -1; + } } else { return -1; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestTimes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestTimes.java index 350b38995c4..918743d02ff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestTimes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestTimes.java @@ -50,4 +50,15 @@ public void testPositiveStartandFinishTimes() { elapsed = Times.elapsed(5, 10, false); Assert.assertEquals("Elapsed time is not 5", 5, elapsed); } + + @Test + public void testFinishTimesAheadOfStartTimes() { + long elapsed = Times.elapsed(10, 5, true); + Assert.assertEquals("Elapsed time is not -1", -1, elapsed); + elapsed = Times.elapsed(10, 5, false); + Assert.assertEquals("Elapsed time is not -1", -1, elapsed); + // use Long.MAX_VALUE to ensure started time is after the current one + elapsed = Times.elapsed(Long.MAX_VALUE, 0, true); + Assert.assertEquals("Elapsed time is not -1", -1, elapsed); + } } \ No newline at end of file From 5644f529f33b49e7da8ce6fe4067c6ad5b3f2b2c Mon Sep 17 00:00:00 2001 From: Sanford Ryza Date: Fri, 4 Jul 2014 15:16:43 +0000 Subject: [PATCH 103/112] YARN-2250. FairScheduler.findLowestCommonAncestorQueue returns null when queues not identical (Krisztian Horvath via Sandy Ryza) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1607872 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../scheduler/fair/FairScheduler.java | 9 +-- .../scheduler/fair/TestFairScheduler.java | 69 +++++++++++++++++++ 3 files changed, 77 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 921ba31d68b..924a8b9b9fc 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -349,6 +349,9 @@ Release 2.5.0 - UNRELEASED YARN-2232. Fixed ResourceManager to allow DelegationToken owners to be able to cancel their own tokens in secure mode. (Varun Vasudev via vinodkv) + YARN-2250. FairScheduler.findLowestCommonAncestorQueue returns null when + queues not identical (Krisztian Horvath via Sandy Ryza) + Release 2.4.1 - 2014-06-23 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 0cbcaae1424..7a4e79c36a4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -18,7 +18,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; -import com.google.common.base.Preconditions; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; @@ -93,6 +92,7 @@ import org.apache.hadoop.yarn.util.resource.Resources; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; /** * A scheduler that schedules resources between a set of queues. The scheduler @@ -1468,8 +1468,9 @@ private void executeMove(SchedulerApplication app, maxRunningEnforcer.updateRunnabilityOnAppRemoval(attempt, oldQueue); } } - - private FSQueue findLowestCommonAncestorQueue(FSQueue queue1, FSQueue queue2) { + + @VisibleForTesting + FSQueue findLowestCommonAncestorQueue(FSQueue queue1, FSQueue queue2) { // Because queue names include ancestors, separated by periods, we can find // the lowest common ancestors by going from the start of the names until // there's a character that doesn't match. @@ -1481,7 +1482,7 @@ private FSQueue findLowestCommonAncestorQueue(FSQueue queue1, FSQueue queue2) { for (int i = 0; i < Math.max(name1.length(), name2.length()); i++) { if (name1.length() <= i || name2.length() <= i || name1.charAt(i) != name2.charAt(i)) { - return queueMgr.getQueue(name1.substring(lastPeriodIndex)); + return queueMgr.getQueue(name1.substring(0, lastPeriodIndex)); } else if (name1.charAt(i) == '.') { lastPeriodIndex = i; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index a54387a9b0c..dbc79d907d3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -25,6 +25,8 @@ import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import java.io.File; import java.io.FileWriter; @@ -3077,4 +3079,71 @@ public void testMoveToNonexistentQueue() throws Exception { createSchedulingRequest(1024, 1, "queue1", "user1", 3); scheduler.moveApplication(appAttId.getApplicationId(), "queue2"); } + + @Test + public void testLowestCommonAncestorForNonRootParent() throws Exception { + scheduler.init(conf); + scheduler.start(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + FSLeafQueue aQueue = mock(FSLeafQueue.class); + FSLeafQueue bQueue = mock(FSLeafQueue.class); + when(aQueue.getName()).thenReturn("root.queue1.a"); + when(bQueue.getName()).thenReturn("root.queue1.b"); + + QueueManager queueManager = scheduler.getQueueManager(); + FSParentQueue queue1 = queueManager.getParentQueue("queue1", true); + queue1.addChildQueue(aQueue); + queue1.addChildQueue(bQueue); + + FSQueue ancestorQueue = + scheduler.findLowestCommonAncestorQueue(aQueue, bQueue); + assertEquals(ancestorQueue, queue1); + } + + @Test + public void testLowestCommonAncestorRootParent() throws Exception { + scheduler.init(conf); + scheduler.start(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + FSLeafQueue aQueue = mock(FSLeafQueue.class); + FSLeafQueue bQueue = mock(FSLeafQueue.class); + when(aQueue.getName()).thenReturn("root.a"); + when(bQueue.getName()).thenReturn("root.b"); + + QueueManager queueManager = scheduler.getQueueManager(); + FSParentQueue queue1 = queueManager.getParentQueue("root", false); + queue1.addChildQueue(aQueue); + queue1.addChildQueue(bQueue); + + FSQueue ancestorQueue = + scheduler.findLowestCommonAncestorQueue(aQueue, bQueue); + assertEquals(ancestorQueue, queue1); + } + + @Test + public void testLowestCommonAncestorDeeperHierarchy() throws Exception { + scheduler.init(conf); + scheduler.start(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + FSQueue aQueue = mock(FSLeafQueue.class); + FSQueue bQueue = mock(FSLeafQueue.class); + FSQueue a1Queue = mock(FSLeafQueue.class); + FSQueue b1Queue = mock(FSLeafQueue.class); + when(a1Queue.getName()).thenReturn("root.queue1.a.a1"); + when(b1Queue.getName()).thenReturn("root.queue1.b.b1"); + when(aQueue.getChildQueues()).thenReturn(Arrays.asList(a1Queue)); + when(bQueue.getChildQueues()).thenReturn(Arrays.asList(b1Queue)); + + QueueManager queueManager = scheduler.getQueueManager(); + FSParentQueue queue1 = queueManager.getParentQueue("queue1", true); + queue1.addChildQueue(aQueue); + queue1.addChildQueue(bQueue); + + FSQueue ancestorQueue = + scheduler.findLowestCommonAncestorQueue(a1Queue, b1Queue); + assertEquals(ancestorQueue, queue1); + } } From 7b5295513dce9768083ae53282013e31d74573c6 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Fri, 4 Jul 2014 17:31:55 +0000 Subject: [PATCH 104/112] HADOOP-10757. KeyProvider KeyVersion should provide the key name. (asuresh via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1607896 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../apache/hadoop/crypto/key/JavaKeyStoreProvider.java | 10 +++++----- .../java/org/apache/hadoop/crypto/key/KeyProvider.java | 8 +++++++- .../org/apache/hadoop/crypto/key/UserProvider.java | 9 +++++---- .../hadoop/crypto/key/kms/KMSClientProvider.java | 9 +++++---- .../org/apache/hadoop/crypto/key/TestKeyProvider.java | 2 +- .../org/apache/hadoop/crypto/key/kms/server/KMS.java | 4 ++-- .../crypto/key/kms/server/KMSServerJSONUtils.java | 2 ++ 8 files changed, 30 insertions(+), 17 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index beaf28d9a48..abbb4ae26ff 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -168,6 +168,9 @@ Trunk (Unreleased) HADOOP-10695. KMSClientProvider should respect a configurable timeout. (yoderme via tucu) + HADOOP-10757. KeyProvider KeyVersion should provide the key name. + (asuresh via tucu) + BUG FIXES HADOOP-9451. Fault single-layer config if node group topology is enabled. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java index 0f22f6343ae..529a21287ce 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java @@ -173,7 +173,7 @@ public KeyVersion getKeyVersion(String versionName) throws IOException { } catch (UnrecoverableKeyException e) { throw new IOException("Can't recover key " + key + " from " + path, e); } - return new KeyVersion(versionName, key.getEncoded()); + return new KeyVersion(getBaseName(versionName), versionName, key.getEncoded()); } finally { readLock.unlock(); } @@ -277,7 +277,7 @@ public KeyVersion createKey(String name, byte[] material, } cache.put(name, meta); String versionName = buildVersionName(name, 0); - return innerSetKeyVersion(versionName, material, meta.getCipher()); + return innerSetKeyVersion(name, versionName, material, meta.getCipher()); } finally { writeLock.unlock(); } @@ -316,7 +316,7 @@ public void deleteKey(String name) throws IOException { } } - KeyVersion innerSetKeyVersion(String versionName, byte[] material, + KeyVersion innerSetKeyVersion(String name, String versionName, byte[] material, String cipher) throws IOException { try { keyStore.setKeyEntry(versionName, new SecretKeySpec(material, cipher), @@ -326,7 +326,7 @@ KeyVersion innerSetKeyVersion(String versionName, byte[] material, e); } changed = true; - return new KeyVersion(versionName, material); + return new KeyVersion(name, versionName, material); } @Override @@ -344,7 +344,7 @@ public KeyVersion rollNewVersion(String name, } int nextVersion = meta.addVersion(); String versionName = buildVersionName(name, nextVersion); - return innerSetKeyVersion(versionName, material, meta.getCipher()); + return innerSetKeyVersion(name, versionName, material, meta.getCipher()); } finally { writeLock.unlock(); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java index 01d7b697ae1..7fd0aa27c39 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java @@ -63,14 +63,20 @@ public abstract class KeyProvider { * The combination of both the key version name and the key material. */ public static class KeyVersion { + private final String name; private final String versionName; private final byte[] material; - protected KeyVersion(String versionName, + protected KeyVersion(String name, String versionName, byte[] material) { + this.name = name; this.versionName = versionName; this.material = material; } + + public String getName() { + return name; + } public String getVersionName() { return versionName; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/UserProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/UserProvider.java index 6cfb46bd719..e09b3f8d432 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/UserProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/UserProvider.java @@ -55,12 +55,13 @@ public boolean isTransient() { } @Override - public synchronized KeyVersion getKeyVersion(String versionName) { + public synchronized KeyVersion getKeyVersion(String versionName) + throws IOException { byte[] bytes = credentials.getSecretKey(new Text(versionName)); if (bytes == null) { return null; } - return new KeyVersion(versionName, bytes); + return new KeyVersion(getBaseName(versionName), versionName, bytes); } @Override @@ -94,7 +95,7 @@ public synchronized KeyVersion createKey(String name, byte[] material, String versionName = buildVersionName(name, 0); credentials.addSecretKey(nameT, meta.serialize()); credentials.addSecretKey(new Text(versionName), material); - return new KeyVersion(versionName, material); + return new KeyVersion(name, versionName, material); } @Override @@ -125,7 +126,7 @@ public synchronized KeyVersion rollNewVersion(String name, credentials.addSecretKey(new Text(name), meta.serialize()); String versionName = buildVersionName(name, nextVersion); credentials.addSecretKey(new Text(versionName), material); - return new KeyVersion(versionName, material); + return new KeyVersion(name, versionName, material); } @Override diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java index c18e8613d08..7d52854845a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java @@ -84,8 +84,9 @@ private static KeyVersion parseJSONKeyVersion(Map valueMap) { byte[] material = (valueMap.containsKey(KMSRESTConstants.MATERIAL_FIELD)) ? Base64.decodeBase64((String) valueMap.get(KMSRESTConstants.MATERIAL_FIELD)) : null; - keyVersion = new KMSKeyVersion((String) - valueMap.get(KMSRESTConstants.VERSION_NAME_FIELD), material); + String versionName = (String)valueMap.get(KMSRESTConstants.VERSION_NAME_FIELD); + String keyName = (String)valueMap.get(KMSRESTConstants.NAME_FIELD); + keyVersion = new KMSKeyVersion(keyName, versionName, material); } return keyVersion; } @@ -362,8 +363,8 @@ private static T call(HttpURLConnection conn, Map jsonOutput, } public static class KMSKeyVersion extends KeyVersion { - public KMSKeyVersion(String versionName, byte[] material) { - super(versionName, material); + public KMSKeyVersion(String keyName, String versionName, byte[] material) { + super(keyName, versionName, material); } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java index 7da16757638..892cec82ff6 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java @@ -64,7 +64,7 @@ public void testParseVersionName() throws Exception { @Test public void testKeyMaterial() throws Exception { byte[] key1 = new byte[]{1,2,3,4}; - KeyProvider.KeyVersion obj = new KeyProvider.KeyVersion("key1@1", key1); + KeyProvider.KeyVersion obj = new KeyProvider.KeyVersion("key1", "key1@1", key1); assertEquals("key1@1", obj.getVersionName()); assertArrayEquals(new byte[]{1,2,3,4}, obj.getMaterial()); } diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java index 3446c787b88..3574bf43b74 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java +++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java @@ -90,8 +90,8 @@ private static void assertAccess(KMSACLs.Type aclType, Principal principal, private static KeyProvider.KeyVersion removeKeyMaterial( KeyProvider.KeyVersion keyVersion) { - return new KMSClientProvider.KMSKeyVersion(keyVersion.getVersionName(), - null); + return new KMSClientProvider.KMSKeyVersion(keyVersion.getName(), + keyVersion.getVersionName(), null); } private static URI getKeyURI(String name) throws URISyntaxException { diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java index 9131a189adb..94501ecf3d4 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java +++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java @@ -35,6 +35,8 @@ public class KMSServerJSONUtils { public static Map toJSON(KeyProvider.KeyVersion keyVersion) { Map json = new LinkedHashMap(); if (keyVersion != null) { + json.put(KMSRESTConstants.NAME_FIELD, + keyVersion.getName()); json.put(KMSRESTConstants.VERSION_NAME_FIELD, keyVersion.getVersionName()); json.put(KMSRESTConstants.MATERIAL_FIELD, keyVersion.getMaterial()); From 405dbd521e88b2762657b924f23e4eb6adbe5843 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Fri, 4 Jul 2014 19:41:00 +0000 Subject: [PATCH 105/112] HADOOP-10719. Add generateEncryptedKey and decryptEncryptedKey methods to KeyProvider. (asuresh via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1607918 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../key/KeyProviderCryptoExtension.java | 246 ++++++++++++++++++ .../crypto/key/KeyProviderExtension.java | 123 +++++++++ .../key/TestKeyProviderCryptoExtension.java | 66 +++++ 4 files changed, 438 insertions(+) create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderExtension.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index abbb4ae26ff..17e8f871477 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -23,6 +23,9 @@ Trunk (Unreleased) Mike Liddell, Chuan Liu, Lengning Liu, Ivan Mitic, Michael Rys, Alexander Stojanovich, Brian Swan, and Min Wei via cnauroth) + HADOOP-10719. Add generateEncryptedKey and decryptEncryptedKey + methods to KeyProvider. (asuresh via tucu) + IMPROVEMENTS HADOOP-8017. Configure hadoop-main pom to get rid of M2E plugin execution diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java new file mode 100644 index 00000000000..4a802ed8f52 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java @@ -0,0 +1,246 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.crypto.key; + +import java.io.IOException; +import java.security.GeneralSecurityException; +import java.security.SecureRandom; + +import javax.crypto.Cipher; +import javax.crypto.spec.IvParameterSpec; +import javax.crypto.spec.SecretKeySpec; + +import com.google.common.base.Preconditions; + +/** + * A KeyProvider with Cytographic Extensions specifically for generating + * Encrypted Keys as well as decrypting them + * + */ +public class KeyProviderCryptoExtension extends + KeyProviderExtension { + + protected static final String EEK = "EEK"; + protected static final String EK = "EK"; + + /** + * This is a holder class whose instance contains the keyVersionName, iv + * used to generate the encrypted Key and the encrypted KeyVersion + */ + public static class EncryptedKeyVersion { + private String keyVersionName; + private byte[] iv; + private KeyVersion encryptedKey; + + protected EncryptedKeyVersion(String keyVersionName, byte[] iv, + KeyVersion encryptedKey) { + this.keyVersionName = keyVersionName; + this.iv = iv; + this.encryptedKey = encryptedKey; + } + + public String getKeyVersionName() { + return keyVersionName; + } + + public byte[] getIv() { + return iv; + } + + public KeyVersion getEncryptedKey() { + return encryptedKey; + } + + } + + /** + * CryptoExtension is a type of Extension that exposes methods to generate + * EncryptedKeys and to decrypt the same. + */ + public interface CryptoExtension extends KeyProviderExtension.Extension { + + /** + * Generates a key material and encrypts it using the given key version name + * and initialization vector. The generated key material is of the same + * length as the KeyVersion material and is encrypted using the + * same cipher. + *

    + * NOTE: The generated key is not stored by the KeyProvider + * + * @param encryptionKeyVersion + * a KeyVersion object containing the keyVersion name and material + * to encrypt. + * @return EncryptedKeyVersion with the generated key material, the version + * name is 'EEK' (for Encrypted Encryption Key) + * @throws IOException + * thrown if the key material could not be generated + * @throws GeneralSecurityException + * thrown if the key material could not be encrypted because of a + * cryptographic issue. + */ + public EncryptedKeyVersion generateEncryptedKey( + KeyVersion encryptionKeyVersion) throws IOException, + GeneralSecurityException; + + /** + * Decrypts an encrypted byte[] key material using the given a key version + * name and initialization vector. + * + * @param encryptedKeyVersion + * contains keyVersionName and IV to decrypt the encrypted key + * material + * @return a KeyVersion with the decrypted key material, the version name is + * 'EK' (For Encryption Key) + * @throws IOException + * thrown if the key material could not be decrypted + * @throws GeneralSecurityException + * thrown if the key material could not be decrypted because of a + * cryptographic issue. + */ + public KeyVersion decryptEncryptedKey( + EncryptedKeyVersion encryptedKeyVersion) throws IOException, + GeneralSecurityException; + } + + private static class DefaultCryptoExtension implements CryptoExtension { + + private final KeyProvider keyProvider; + + private DefaultCryptoExtension(KeyProvider keyProvider) { + this.keyProvider = keyProvider; + } + + // the IV used to encrypt a EK typically will be the same IV used to + // encrypt data with the EK. To avoid any chance of weakening the + // encryption because the same IV is used, we simply XOR the IV thus we + // are not using the same IV for 2 different encryptions (even if they + // are done using different keys) + private byte[] flipIV(byte[] iv) { + byte[] rIv = new byte[iv.length]; + for (int i = 0; i < iv.length; i++) { + rIv[i] = (byte) (iv[i] ^ 0xff); + } + return rIv; + } + + @Override + public EncryptedKeyVersion generateEncryptedKey(KeyVersion keyVersion) + throws IOException, GeneralSecurityException { + KeyVersion keyVer = + keyProvider.getKeyVersion(keyVersion.getVersionName()); + Preconditions.checkNotNull(keyVer, "KeyVersion name '%s' does not exist", + keyVersion.getVersionName()); + byte[] newKey = new byte[keyVer.getMaterial().length]; + SecureRandom.getInstance("SHA1PRNG").nextBytes(newKey); + Cipher cipher = Cipher.getInstance("AES/CTR/NoPadding"); + byte[] iv = SecureRandom.getSeed(cipher.getBlockSize()); + cipher.init(Cipher.ENCRYPT_MODE, new SecretKeySpec(keyVer.getMaterial(), + "AES"), new IvParameterSpec(flipIV(iv))); + byte[] ek = cipher.doFinal(newKey); + return new EncryptedKeyVersion(keyVersion.getVersionName(), iv, + new KeyVersion(keyVer.getName(), EEK, ek)); + } + + @Override + public KeyVersion decryptEncryptedKey( + EncryptedKeyVersion encryptedKeyVersion) throws IOException, + GeneralSecurityException { + KeyVersion keyVer = + keyProvider.getKeyVersion(encryptedKeyVersion.getKeyVersionName()); + Preconditions.checkNotNull(keyVer, "KeyVersion name '%s' does not exist", + encryptedKeyVersion.getKeyVersionName()); + KeyVersion keyVersion = encryptedKeyVersion.getEncryptedKey(); + Cipher cipher = Cipher.getInstance("AES/CTR/NoPadding"); + cipher.init(Cipher.DECRYPT_MODE, + new SecretKeySpec(keyVersion.getMaterial(), "AES"), + new IvParameterSpec(flipIV(encryptedKeyVersion.getIv()))); + byte[] ek = + cipher.doFinal(encryptedKeyVersion.getEncryptedKey().getMaterial()); + return new KeyVersion(keyVer.getName(), EK, ek); + } + + } + + private KeyProviderCryptoExtension(KeyProvider keyProvider, + CryptoExtension extension) { + super(keyProvider, extension); + } + + /** + * Generates a key material and encrypts it using the given key version name + * and initialization vector. The generated key material is of the same + * length as the KeyVersion material and is encrypted using the + * same cipher. + *

    + * NOTE: The generated key is not stored by the KeyProvider + * + * @param encryptionKey a KeyVersion object containing the keyVersion name and + * material to encrypt. + * @return EncryptedKeyVersion with the generated key material, the version + * name is 'EEK' (for Encrypted Encryption Key) + * @throws IOException thrown if the key material could not be generated + * @throws GeneralSecurityException thrown if the key material could not be + * encrypted because of a cryptographic issue. + */ + public EncryptedKeyVersion generateEncryptedKey(KeyVersion encryptionKey) + throws IOException, + GeneralSecurityException { + return getExtension().generateEncryptedKey(encryptionKey); + } + + /** + * Decrypts an encrypted byte[] key material using the given a key version + * name and initialization vector. + * + * @param encryptedKey contains keyVersionName and IV to decrypt the encrypted + * key material + * @return a KeyVersion with the decrypted key material, the version name is + * 'EK' (For Encryption Key) + * @throws IOException thrown if the key material could not be decrypted + * @throws GeneralSecurityException thrown if the key material could not be + * decrypted because of a cryptographic issue. + */ + public KeyVersion decryptEncryptedKey(EncryptedKeyVersion encryptedKey) + throws IOException, GeneralSecurityException { + return getExtension().decryptEncryptedKey(encryptedKey); + } + + /** + * Creates a KeyProviderCryptoExtension using a given + * {@link KeyProvider}. + *

    + * If the given KeyProvider implements the + * {@link CryptoExtension} interface the KeyProvider itself + * will provide the extension functionality, otherwise a default extension + * implementation will be used. + * + * @param keyProvider KeyProvider to use to create the + * KeyProviderCryptoExtension extension. + * @return a KeyProviderCryptoExtension instance using the + * given KeyProvider. + */ + public static KeyProviderCryptoExtension createKeyProviderCryptoExtension( + KeyProvider keyProvider) { + CryptoExtension cryptoExtension = (keyProvider instanceof CryptoExtension) + ? (CryptoExtension) keyProvider + : new DefaultCryptoExtension(keyProvider); + return new KeyProviderCryptoExtension(keyProvider, cryptoExtension); + } + +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderExtension.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderExtension.java new file mode 100644 index 00000000000..9e1a1853868 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderExtension.java @@ -0,0 +1,123 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.crypto.key; + +import java.io.IOException; +import java.security.NoSuchAlgorithmException; +import java.util.List; + +/** + * This is a utility class used to extend the functionality of KeyProvider, that + * takes a KeyProvider and an Extension. It implements all the required methods + * of the KeyProvider by delegating it to the provided KeyProvider. + */ +public abstract class KeyProviderExtension + extends KeyProvider { + + /** + * A marker interface for the KeyProviderExtension subclass implement. + */ + public static interface Extension { + } + + private KeyProvider keyProvider; + private E extension; + + public KeyProviderExtension(KeyProvider keyProvider, E extensions) { + this.keyProvider = keyProvider; + this.extension = extensions; + } + + protected E getExtension() { + return extension; + } + + protected KeyProvider getKeyProvider() { + return keyProvider; + } + + @Override + public boolean isTransient() { + return keyProvider.isTransient(); + } + + @Override + public Metadata[] getKeysMetadata(String... names) throws IOException { + return keyProvider.getKeysMetadata(names); + } + + @Override + public KeyVersion getCurrentKey(String name) throws IOException { + return keyProvider.getCurrentKey(name); + } + + @Override + public KeyVersion createKey(String name, Options options) + throws NoSuchAlgorithmException, IOException { + return keyProvider.createKey(name, options); + } + + @Override + public KeyVersion rollNewVersion(String name) + throws NoSuchAlgorithmException, IOException { + return keyProvider.rollNewVersion(name); + } + + @Override + public KeyVersion getKeyVersion(String versionName) throws IOException { + return keyProvider.getKeyVersion(versionName); + } + + @Override + public List getKeys() throws IOException { + return keyProvider.getKeys(); + } + + @Override + public List getKeyVersions(String name) throws IOException { + return keyProvider.getKeyVersions(name); + } + + @Override + public Metadata getMetadata(String name) throws IOException { + return keyProvider.getMetadata(name); + } + + @Override + public KeyVersion createKey(String name, byte[] material, Options options) + throws IOException { + return keyProvider.createKey(name, material, options); + } + + @Override + public void deleteKey(String name) throws IOException { + keyProvider.deleteKey(name); + } + + @Override + public KeyVersion rollNewVersion(String name, byte[] material) + throws IOException { + return keyProvider.rollNewVersion(name, material); + } + + @Override + public void flush() throws IOException { + keyProvider.flush(); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java new file mode 100644 index 00000000000..d8da557d40e --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.crypto.key; + +import org.apache.hadoop.conf.Configuration; +import org.junit.Assert; +import org.junit.Test; + +import java.net.URI; +import java.security.SecureRandom; + +public class TestKeyProviderCryptoExtension { + + private static final String CIPHER = "AES"; + + @Test + public void testGenerateEncryptedKey() throws Exception { + Configuration conf = new Configuration(); + KeyProvider kp = + new UserProvider.Factory().createProvider(new URI("user:///"), conf); + KeyProvider.Options options = new KeyProvider.Options(conf); + options.setCipher(CIPHER); + options.setBitLength(128); + KeyProvider.KeyVersion kv = kp.createKey("foo", SecureRandom.getSeed(16), + options); + KeyProviderCryptoExtension kpExt = + KeyProviderCryptoExtension.createKeyProviderCryptoExtension(kp); + + KeyProviderCryptoExtension.EncryptedKeyVersion ek1 = + kpExt.generateEncryptedKey(kv); + Assert.assertEquals(KeyProviderCryptoExtension.EEK, + ek1.getEncryptedKey().getVersionName()); + Assert.assertNotNull(ek1.getEncryptedKey().getMaterial()); + Assert.assertEquals(kv.getMaterial().length, + ek1.getEncryptedKey().getMaterial().length); + KeyProvider.KeyVersion k1 = kpExt.decryptEncryptedKey(ek1); + Assert.assertEquals(KeyProviderCryptoExtension.EK, k1.getVersionName()); + KeyProvider.KeyVersion k1a = kpExt.decryptEncryptedKey(ek1); + Assert.assertArrayEquals(k1.getMaterial(), k1a.getMaterial()); + Assert.assertEquals(kv.getMaterial().length, k1.getMaterial().length); + + KeyProviderCryptoExtension.EncryptedKeyVersion ek2 = + kpExt.generateEncryptedKey(kv); + KeyProvider.KeyVersion k2 = kpExt.decryptEncryptedKey(ek2); + boolean eq = true; + for (int i = 0; eq && i < ek2.getEncryptedKey().getMaterial().length; i++) { + eq = k2.getMaterial()[i] == k1.getMaterial()[i]; + } + Assert.assertFalse(eq); + } +} From ed642b5d8dd4cbe19518f0d36dc63700bc1bd7a9 Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Sun, 6 Jul 2014 19:16:45 +0000 Subject: [PATCH 106/112] HADOOP-10769. Create KeyProvider extension to handle delegation tokens. Contributed by Arun Suresh. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1608286 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../KeyProviderDelegationTokenExtension.java | 111 ++++++++++++++++++ ...stKeyProviderDelegationTokenExtension.java | 66 +++++++++++ 3 files changed, 180 insertions(+) create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderDelegationTokenExtension.java diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 17e8f871477..1c6b77eb9ec 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -174,6 +174,9 @@ Trunk (Unreleased) HADOOP-10757. KeyProvider KeyVersion should provide the key name. (asuresh via tucu) + HADOOP-10769. Create KeyProvider extension to handle delegation tokens. + (Arun Suresh via atm) + BUG FIXES HADOOP-9451. Fault single-layer config if node group topology is enabled. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java new file mode 100644 index 00000000000..ccf382241ff --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java @@ -0,0 +1,111 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.crypto.key; + +import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.token.Token; + +/** + * A KeyProvider extension with the ability to add a renewer's Delegation + * Tokens to the provided Credentials. + */ +public class KeyProviderDelegationTokenExtension extends + KeyProviderExtension + { + + private static DelegationTokenExtension DEFAULT_EXTENSION = + new DefaultDelegationTokenExtension(); + + /** + * DelegationTokenExtension is a type of Extension that exposes methods to + * needed to work with Delegation Tokens. + */ + public interface DelegationTokenExtension extends + KeyProviderExtension.Extension { + + /** + * The implementer of this class will take a renewer and add all + * delegation tokens associated with the renewer to the + * Credentials object if it is not already present, + * @param renewer the user allowed to renew the delegation tokens + * @param credentials cache in which to add new delegation tokens + * @return list of new delegation tokens + */ + public Token[] addDelegationTokens(final String renewer, + Credentials credentials); + } + + /** + * Default implementation of {@link DelegationTokenExtension} that + * implements the method as a no-op. + */ + private static class DefaultDelegationTokenExtension implements + DelegationTokenExtension { + + @Override + public Token[] addDelegationTokens(String renewer, + Credentials credentials) { + return null; + } + + } + + private KeyProviderDelegationTokenExtension(KeyProvider keyProvider, + DelegationTokenExtension extensions) { + super(keyProvider, extensions); + } + + /** + * Passes the renewer and Credentials object to the underlying + * {@link DelegationTokenExtension} + * @param renewer the user allowed to renew the delegation tokens + * @param credentials cache in which to add new delegation tokens + * @return list of new delegation tokens + */ + public Token[] addDelegationTokens(final String renewer, + Credentials credentials) { + return getExtension().addDelegationTokens(renewer, credentials); + } + + /** + * Creates a KeyProviderDelegationTokenExtension using a given + * {@link KeyProvider}. + *

    + * If the given KeyProvider implements the + * {@link DelegationTokenExtension} interface the KeyProvider + * itself will provide the extension functionality, otherwise a default + * extension implementation will be used. + * + * @param keyProvider KeyProvider to use to create the + * KeyProviderDelegationTokenExtension extension. + * @return a KeyProviderDelegationTokenExtension instance + * using the given KeyProvider. + */ + public static KeyProviderDelegationTokenExtension + createKeyProviderDelegationTokenExtension(KeyProvider keyProvider) { + + DelegationTokenExtension delTokExtension = + (keyProvider instanceof DelegationTokenExtension) ? + (DelegationTokenExtension) keyProvider : + DEFAULT_EXTENSION; + return new KeyProviderDelegationTokenExtension( + keyProvider, delTokExtension); + + } + +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderDelegationTokenExtension.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderDelegationTokenExtension.java new file mode 100644 index 00000000000..52dedf00512 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderDelegationTokenExtension.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.crypto.key; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.net.URI; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension.DelegationTokenExtension; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.token.Token; +import org.junit.Assert; +import org.junit.Test; + +public class TestKeyProviderDelegationTokenExtension { + + public static abstract class MockKeyProvider extends + KeyProvider implements DelegationTokenExtension { + } + + @Test + public void testCreateExtension() throws Exception { + Configuration conf = new Configuration(); + Credentials credentials = new Credentials(); + KeyProvider kp = + new UserProvider.Factory().createProvider(new URI("user:///"), conf); + KeyProviderDelegationTokenExtension kpDTE1 = + KeyProviderDelegationTokenExtension + .createKeyProviderDelegationTokenExtension(kp); + Assert.assertNotNull(kpDTE1); + // Default implementation should be a no-op and return null + Assert.assertNull(kpDTE1.addDelegationTokens("user", credentials)); + + MockKeyProvider mock = mock(MockKeyProvider.class); + when(mock.addDelegationTokens("renewer", credentials)).thenReturn( + new Token[] { new Token(null, null, new Text("kind"), new Text( + "service")) }); + KeyProviderDelegationTokenExtension kpDTE2 = + KeyProviderDelegationTokenExtension + .createKeyProviderDelegationTokenExtension(mock); + Token[] tokens = + kpDTE2.addDelegationTokens("renewer", credentials); + Assert.assertNotNull(tokens); + Assert.assertEquals("kind", tokens[0].getKind().toString()); + + } + +} From 6d7dbd4fedd91a5177a2f0d90c5822394ab18529 Mon Sep 17 00:00:00 2001 From: Jian He Date: Mon, 7 Jul 2014 04:37:59 +0000 Subject: [PATCH 107/112] YARN-1367. Changed NM to not kill containers on NM resync if RM work-preserving restart is enabled. Contributed by Anubhav Dhoot git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1608334 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../yarn/server/nodemanager/NodeManager.java | 15 ++- .../nodemanager/TestNodeManagerResync.java | 105 +++++++++++++----- 3 files changed, 91 insertions(+), 32 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 924a8b9b9fc..d2be7bc88d7 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -72,6 +72,9 @@ Release 2.5.0 - UNRELEASED YARN-1713. Added get-new-app and submit-app functionality to RM web services. (Varun Vasudev via vinodkv) + YARN-1367. Changed NM to not kill containers on NM resync if RM work-preserving + restart is enabled. (Anubhav Dhoot via jianhe) + IMPROVEMENTS YARN-1479. Invalid NaN values in Hadoop REST API JSON response (Chen He via diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java index 2292a0dc9de..1109b087c17 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java @@ -84,7 +84,8 @@ public class NodeManager extends CompositeService private NMStateStoreService nmStore = null; private AtomicBoolean isStopping = new AtomicBoolean(false); - + private boolean rmWorkPreservingRestartEnabled; + public NodeManager() { super(NodeManager.class.getName()); } @@ -173,6 +174,10 @@ protected void serviceInit(Configuration conf) throws Exception { conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true); + rmWorkPreservingRestartEnabled = conf.getBoolean(YarnConfiguration + .RM_WORK_PRESERVING_RECOVERY_ENABLED, + YarnConfiguration.DEFAULT_RM_WORK_PRESERVING_RECOVERY_ENABLED); + initAndStartRecoveryStore(conf); NMContainerTokenSecretManager containerTokenSecretManager = @@ -276,8 +281,12 @@ public void run() { try { LOG.info("Notifying ContainerManager to block new container-requests"); containerManager.setBlockNewContainerRequests(true); - LOG.info("Cleaning up running containers on resync"); - containerManager.cleanupContainersOnNMResync(); + if (!rmWorkPreservingRestartEnabled) { + LOG.info("Cleaning up running containers on resync"); + containerManager.cleanupContainersOnNMResync(); + } else { + LOG.info("Preserving containers on resync"); + } ((NodeStatusUpdaterImpl) nodeStatusUpdater) .rebootNodeStatusUpdaterAndRegisterWithRM(); } catch (YarnRuntimeException e) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java index ea8f7343ec0..bd531865815 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java @@ -59,7 +59,6 @@ import org.apache.hadoop.yarn.server.api.records.NodeAction; import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl; import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.utils.YarnServerBuilderUtils; @@ -85,6 +84,9 @@ public class TestNodeManagerResync { private CyclicBarrier syncBarrier; private AtomicBoolean assertionFailedInThread = new AtomicBoolean(false); private AtomicBoolean isNMShutdownCalled = new AtomicBoolean(false); + private final NodeManagerEvent resyncEvent = + new NodeManagerEvent(NodeManagerEventType.RESYNC); + @Before public void setup() throws UnsupportedFileSystemException { @@ -102,34 +104,56 @@ public void tearDown() throws IOException, InterruptedException { assertionFailedInThread.set(false); } - @SuppressWarnings("unchecked") @Test public void testKillContainersOnResync() throws IOException, InterruptedException, YarnException { - NodeManager nm = new TestNodeManager1(); + TestNodeManager1 nm = new TestNodeManager1(false); + + testContainerPreservationOnResyncImpl(nm, false); + } + + @Test + public void testPreserveContainersOnResyncKeepingContainers() throws + IOException, + InterruptedException, YarnException { + TestNodeManager1 nm = new TestNodeManager1(true); + + testContainerPreservationOnResyncImpl(nm, true); + } + + @SuppressWarnings("unchecked") + protected void testContainerPreservationOnResyncImpl(TestNodeManager1 nm, + boolean isWorkPreservingRestartEnabled) + throws IOException, YarnException, InterruptedException { YarnConfiguration conf = createNMConfig(); - nm.init(conf); - nm.start(); - ContainerId cId = TestNodeManagerShutdown.createContainerId(); - TestNodeManagerShutdown.startContainer(nm, cId, localFS, tmpDir, - processStartFile); + conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, + isWorkPreservingRestartEnabled); - Assert.assertEquals(1, ((TestNodeManager1) nm).getNMRegistrationCount()); - nm.getNMDispatcher().getEventHandler(). - handle( new NodeManagerEvent(NodeManagerEventType.RESYNC)); try { - syncBarrier.await(); - } catch (BrokenBarrierException e) { - } - Assert.assertEquals(2, ((TestNodeManager1) nm).getNMRegistrationCount()); - // Only containers should be killed on resync, apps should lie around. That - // way local resources for apps can be used beyond resync without - // relocalization - Assert.assertTrue(nm.getNMContext().getApplications() - .containsKey(cId.getApplicationAttemptId().getApplicationId())); - Assert.assertFalse(assertionFailedInThread.get()); + nm.init(conf); + nm.start(); + ContainerId cId = TestNodeManagerShutdown.createContainerId(); + TestNodeManagerShutdown.startContainer(nm, cId, localFS, tmpDir, + processStartFile); - nm.stop(); + nm.setExistingContainerId(cId); + Assert.assertEquals(1, ((TestNodeManager1) nm).getNMRegistrationCount()); + nm.getNMDispatcher().getEventHandler().handle(resyncEvent); + try { + syncBarrier.await(); + } catch (BrokenBarrierException e) { + } + Assert.assertEquals(2, ((TestNodeManager1) nm).getNMRegistrationCount()); + // Only containers should be killed on resync, apps should lie around. + // That way local resources for apps can be used beyond resync without + // relocalization + Assert.assertTrue(nm.getNMContext().getApplications() + .containsKey(cId.getApplicationAttemptId().getApplicationId())); + Assert.assertFalse(assertionFailedInThread.get()); + } + finally { + nm.stop(); + } } // This test tests new container requests are blocked when NM starts from @@ -157,7 +181,7 @@ public void testBlockNewContainerRequestsOnStartAndResync() Assert.assertFalse(assertionFailedInThread.get()); nm.stop(); } - + @SuppressWarnings("unchecked") @Test(timeout=10000) public void testNMshutdownWhenResyncThrowException() throws IOException, @@ -169,7 +193,7 @@ public void testNMshutdownWhenResyncThrowException() throws IOException, Assert.assertEquals(1, ((TestNodeManager3) nm).getNMRegistrationCount()); nm.getNMDispatcher().getEventHandler() .handle(new NodeManagerEvent(NodeManagerEventType.RESYNC)); - + synchronized (isNMShutdownCalled) { while (isNMShutdownCalled.get() == false) { try { @@ -178,7 +202,7 @@ public void testNMshutdownWhenResyncThrowException() throws IOException, } } } - + Assert.assertTrue("NM shutdown not called.",isNMShutdownCalled.get()); nm.stop(); } @@ -313,6 +337,16 @@ private YarnConfiguration createNMConfig() { class TestNodeManager1 extends NodeManager { private int registrationCount = 0; + private boolean containersShouldBePreserved; + private ContainerId existingCid; + + public TestNodeManager1(boolean containersShouldBePreserved) { + this.containersShouldBePreserved = containersShouldBePreserved; + } + + public void setExistingContainerId(ContainerId cId) { + existingCid = cId; + } @Override protected NodeStatusUpdater createNodeStatusUpdater(Context context, @@ -344,10 +378,23 @@ protected void rebootNodeStatusUpdaterAndRegisterWithRM() { .containermanager.container.Container> containers = getNMContext().getContainers(); try { - // ensure that containers are empty before restart nodeStatusUpdater - Assert.assertTrue(containers.isEmpty()); - super.rebootNodeStatusUpdaterAndRegisterWithRM(); - syncBarrier.await(); + try { + if (containersShouldBePreserved) { + Assert.assertFalse(containers.isEmpty()); + Assert.assertTrue(containers.containsKey(existingCid)); + } else { + // ensure that containers are empty before restart nodeStatusUpdater + Assert.assertTrue(containers.isEmpty()); + } + super.rebootNodeStatusUpdaterAndRegisterWithRM(); + } + catch (AssertionError ae) { + ae.printStackTrace(); + assertionFailedInThread.set(true); + } + finally { + syncBarrier.await(); + } } catch (InterruptedException e) { } catch (BrokenBarrierException e) { } catch (AssertionError ae) { From 93c1bc8114daf1230cc583aed851e122fc87d035 Mon Sep 17 00:00:00 2001 From: Chris Nauroth Date: Mon, 7 Jul 2014 17:37:04 +0000 Subject: [PATCH 108/112] HDFS-6617. Flake TestDFSZKFailoverController.testManualFailoverWithDFSHAAdmin due to a long edit log sync op. Contributed by Liang Xie. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1608522 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../server/namenode/ha/TestDFSZKFailoverController.java | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 0f04bc18795..6336d9cb6ed 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -270,6 +270,9 @@ Release 2.6.0 - UNRELEASED BUG FIXES + HDFS-6617. Flake TestDFSZKFailoverController.testManualFailoverWithDFSHAAdmin + due to a long edit log sync op. (Liang Xie via cnauroth) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDFSZKFailoverController.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDFSZKFailoverController.java index 18972655765..bcbd5432dfa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDFSZKFailoverController.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDFSZKFailoverController.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.MiniDFSNNTopology; +import org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.tools.DFSHAAdmin; import org.apache.hadoop.hdfs.tools.DFSZKFailoverController; @@ -53,6 +54,11 @@ public class TestDFSZKFailoverController extends ClientBaseWithFixes { private TestContext ctx; private ZKFCThread thr1, thr2; private FileSystem fs; + + static { + // Make tests run faster by avoiding fsync() + EditLogFileOutputStream.setShouldSkipFsyncForTesting(true); + } @Before public void setup() throws Exception { From 55e7ddf514b45bb45f2cfa1995c0d4b786ae0df3 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Mon, 7 Jul 2014 18:10:42 +0000 Subject: [PATCH 109/112] HADOOP-10782. Fix typo in DataChecksum class. Contributed by Jingguo Yao. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1608539 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++ .../src/main/java/org/apache/hadoop/util/DataChecksum.java | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 1c6b77eb9ec..034686ac512 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -523,6 +523,8 @@ Release 2.5.0 - UNRELEASED HADOOP-10312 Shell.ExitCodeException to have more useful toString (stevel) + HADOOP-10782. Fix typo in DataChecksum class. (Jingguo Yao via suresh) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java index 63c42d28198..c50ecfd64ed 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java @@ -30,7 +30,7 @@ import org.apache.hadoop.fs.ChecksumException; /** - * This class provides inteface and utilities for processing checksums for + * This class provides interface and utilities for processing checksums for * DFS data transfers. */ @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"}) From d85f90e5b35f1eb1aa359688ff15870c8234d683 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Mon, 7 Jul 2014 19:45:57 +0000 Subject: [PATCH 110/112] MAPREDUCE-5868. Fixed an issue with TestPipeApplication that was causing the nightly builds to fail. Contributed by Akira Ajisaka. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1608579 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../org/apache/hadoop/mapred/pipes/TestPipeApplication.java | 2 ++ 2 files changed, 5 insertions(+) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 37d69eb2fbd..ffe834fa496 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -291,6 +291,9 @@ Release 2.5.0 - UNRELEASED MAPREDUCE-5900. Changed to the interpret container preemption exit code as a task attempt killing event. (Mayank Bansal via zjshen) + MAPREDUCE-5868. Fixed an issue with TestPipeApplication that was causing the + nightly builds to fail. (Akira Ajisaka via vinodkv) + Release 2.4.1 - 2014-06-23 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java index 04afa15ffa1..69994f36bd2 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipeApplication.java @@ -183,6 +183,8 @@ public void testApplication() throws Throwable { output.setWriter(wr); conf.set(Submitter.PRESERVE_COMMANDFILE, "true"); + initStdOut(conf); + Application, Writable, IntWritable, Text> application = new Application, Writable, IntWritable, Text>( conf, rReader, output, reporter, IntWritable.class, Text.class); application.getDownlink().flush(); From 286ae1edd09987444644c13dec3b55ab783ec33e Mon Sep 17 00:00:00 2001 From: Chris Nauroth Date: Mon, 7 Jul 2014 19:56:44 +0000 Subject: [PATCH 111/112] MAPREDUCE-5866. TestFixedLengthInputFormat fails in windows. Contributed by Varun Vasudev. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1608585 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../hadoop/mapred/TestFixedLengthInputFormat.java | 9 ++++++--- .../lib/input/TestFixedLengthInputFormat.java | 15 +++++++++------ 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index ffe834fa496..e9732b8922a 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -157,6 +157,9 @@ Release 2.6.0 - UNRELEASED BUG FIXES + MAPREDUCE-5866. TestFixedLengthInputFormat fails in windows. + (Varun Vasudev via cnauroth) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFixedLengthInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFixedLengthInputFormat.java index c9228424486..8013feb1ba2 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFixedLengthInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFixedLengthInputFormat.java @@ -372,10 +372,13 @@ private static List readSplit(FixedLengthInputFormat format, format.getRecordReader(split, job, voidReporter); LongWritable key = reader.createKey(); BytesWritable value = reader.createValue(); - while (reader.next(key, value)) { - result.add(new String(value.getBytes(), 0, value.getLength())); + try { + while (reader.next(key, value)) { + result.add(new String(value.getBytes(), 0, value.getLength())); + } + } finally { + reader.close(); } - reader.close(); return result; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFixedLengthInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFixedLengthInputFormat.java index dc3a1db9ac8..b82b495f1cb 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFixedLengthInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFixedLengthInputFormat.java @@ -417,15 +417,18 @@ private static List readSplit(FixedLengthInputFormat format, new MapContextImpl(job.getConfiguration(), context.getTaskAttemptID(), reader, null, null, MapReduceTestUtil.createDummyReporter(), split); - reader.initialize(split, mcontext); LongWritable key; BytesWritable value; - while (reader.nextKeyValue()) { - key = reader.getCurrentKey(); - value = reader.getCurrentValue(); - result.add(new String(value.getBytes(), 0, value.getLength())); + try { + reader.initialize(split, mcontext); + while (reader.nextKeyValue()) { + key = reader.getCurrentKey(); + value = reader.getCurrentValue(); + result.add(new String(value.getBytes(), 0, value.getLength())); + } + } finally { + reader.close(); } - reader.close(); return result; } From 4b2ded8202f9d07ba44066650dc4e5c987cbbacc Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Mon, 7 Jul 2014 20:14:01 +0000 Subject: [PATCH 112/112] MAPREDUCE-5517. Fixed MapReduce ApplicationMaster to not validate reduce side resource configuration for deciding uber-mode on map-only jobs. Contributed by Siqi Li. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1608595 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 4 +++ .../mapreduce/v2/app/job/impl/JobImpl.java | 33 ++++++++++--------- .../v2/app/job/impl/TestJobImpl.java | 9 +++++ 3 files changed, 31 insertions(+), 15 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index e9732b8922a..449c93c5e05 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -297,6 +297,10 @@ Release 2.5.0 - UNRELEASED MAPREDUCE-5868. Fixed an issue with TestPipeApplication that was causing the nightly builds to fail. (Akira Ajisaka via vinodkv) + MAPREDUCE-5517. Fixed MapReduce ApplicationMaster to not validate reduce side + resource configuration for deciding uber-mode on map-only jobs. (Siqi Li via + vinodkv) + Release 2.4.1 - 2014-06-23 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java index f8b1fd0a514..19dbb05c714 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java @@ -1218,22 +1218,25 @@ private void makeUberDecision(long dataInputLength) { boolean smallNumReduceTasks = (numReduceTasks <= sysMaxReduces); boolean smallInput = (dataInputLength <= sysMaxBytes); // ignoring overhead due to UberAM and statics as negligible here: + long requiredMapMB = conf.getLong(MRJobConfig.MAP_MEMORY_MB, 0); + long requiredReduceMB = conf.getLong(MRJobConfig.REDUCE_MEMORY_MB, 0); + long requiredMB = Math.max(requiredMapMB, requiredReduceMB); + int requiredMapCores = conf.getInt( + MRJobConfig.MAP_CPU_VCORES, + MRJobConfig.DEFAULT_MAP_CPU_VCORES); + int requiredReduceCores = conf.getInt( + MRJobConfig.REDUCE_CPU_VCORES, + MRJobConfig.DEFAULT_REDUCE_CPU_VCORES); + int requiredCores = Math.max(requiredMapCores, requiredReduceCores); + if (numReduceTasks == 0) { + requiredMB = requiredMapMB; + requiredCores = requiredMapCores; + } boolean smallMemory = - ( (Math.max(conf.getLong(MRJobConfig.MAP_MEMORY_MB, 0), - conf.getLong(MRJobConfig.REDUCE_MEMORY_MB, 0)) - <= sysMemSizeForUberSlot) - || (sysMemSizeForUberSlot == JobConf.DISABLED_MEMORY_LIMIT)); - boolean smallCpu = - ( - Math.max( - conf.getInt( - MRJobConfig.MAP_CPU_VCORES, - MRJobConfig.DEFAULT_MAP_CPU_VCORES), - conf.getInt( - MRJobConfig.REDUCE_CPU_VCORES, - MRJobConfig.DEFAULT_REDUCE_CPU_VCORES)) - <= sysCPUSizeForUberSlot - ); + (requiredMB <= sysMemSizeForUberSlot) + || (sysMemSizeForUberSlot == JobConf.DISABLED_MEMORY_LIMIT); + + boolean smallCpu = requiredCores <= sysCPUSizeForUberSlot; boolean notChainJob = !isChainJob(conf); // User has overall veto power over uberization, or user can modify diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java index 6cfc83ea006..65352b75ad0 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java @@ -657,6 +657,15 @@ public void testUberDecision() throws Exception { conf.setInt(MRJobConfig.JOB_UBERTASK_MAXMAPS, 1); isUber = testUberDecision(conf); Assert.assertFalse(isUber); + + // enable uber mode of 0 reducer no matter how much memory assigned to reducer + conf = new Configuration(); + conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, true); + conf.setInt(MRJobConfig.NUM_REDUCES, 0); + conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, 2048); + conf.setInt(MRJobConfig.REDUCE_CPU_VCORES, 10); + isUber = testUberDecision(conf); + Assert.assertTrue(isUber); } private boolean testUberDecision(Configuration conf) {