From 3a3e0f573129c8308332d4b301a9319ee579d85a Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Tue, 30 Apr 2013 21:05:20 +0000 Subject: [PATCH] HDFS-4760. Update inodeMap after node replacement. Contributed by Jing Zhao git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2802@1477827 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES.HDFS-2802.txt | 2 + .../server/blockmanagement/BlockManager.java | 6 - .../hdfs/server/namenode/FSDirectory.java | 197 +++++++----------- .../hdfs/server/namenode/FSEditLogLoader.java | 8 +- .../hdfs/server/namenode/FSImageFormat.java | 2 +- .../hdfs/server/namenode/FSNamesystem.java | 19 +- .../hadoop/hdfs/server/namenode/INode.java | 36 ++-- .../hdfs/server/namenode/INodeDirectory.java | 91 ++++---- .../hdfs/server/namenode/INodeFile.java | 29 +-- .../namenode/INodeFileUnderConstruction.java | 17 +- .../hadoop/hdfs/server/namenode/INodeMap.java | 137 ++++++++++++ .../hdfs/server/namenode/INodeReference.java | 11 +- .../hdfs/server/namenode/INodeSymlink.java | 11 +- .../namenode/INodeWithAdditionalFields.java | 6 +- .../hdfs/server/namenode/INodesInPath.java | 5 - .../snapshot/INodeDirectorySnapshottable.java | 17 +- .../snapshot/INodeDirectoryWithSnapshot.java | 33 +-- ...NodeFileUnderConstructionWithSnapshot.java | 6 +- .../snapshot/INodeFileWithSnapshot.java | 7 +- .../namenode/snapshot/SnapshotManager.java | 5 +- .../hdfs/server/namenode/TestINodeFile.java | 2 +- .../snapshot/TestRenameWithSnapshots.java | 22 +- 22 files changed, 411 insertions(+), 258 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-2802.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-2802.txt index 655363ca768..45da1f61cbe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-2802.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-2802.txt @@ -324,3 +324,5 @@ Branch-2802 Snapshot (Unreleased) HDFS-4773. Fix bugs in quota usage computation and OfflineImageViewer. (Jing Zhao via szetszwo) + + HDFS-4760. Update inodeMap after node replacement. (Jing Zhao via szetszwo) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 8abde8053f7..b83eaa4fc4b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -3016,12 +3016,6 @@ public BlockInfo addBlockCollection(BlockInfo block, BlockCollection bc) { return blocksMap.addBlockCollection(block, bc); } - public void addBlockCollection(BlockCollection bc) { - for(BlockInfo block : bc.getBlocks()) { - addBlockCollection(block, bc); - } - } - public BlockCollection getBlockCollection(Block b) { return blocksMap.getBlockCollection(b); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index e4dab0ea4b3..0cdad8d68df 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@ -61,10 +61,8 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState; -import org.apache.hadoop.hdfs.server.namenode.Content.CountsMap; import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo; import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithCount; -import org.apache.hadoop.hdfs.server.namenode.Quota.Counts; import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable; import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectoryWithSnapshot; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; @@ -73,8 +71,6 @@ import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotException; import org.apache.hadoop.hdfs.util.ByteArray; import org.apache.hadoop.hdfs.util.ReadOnlyList; -import org.apache.hadoop.hdfs.util.GSet; -import org.apache.hadoop.hdfs.util.LightWeightGSet; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -116,7 +112,7 @@ private static INodeDirectoryWithQuota createRoot(FSNamesystem namesystem) { private final int maxComponentLength; private final int maxDirItems; private final int lsLimit; // max list limit - private GSet inodeMap; // Synchronized by dirLock + private final INodeMap inodeMap; // Synchronized by dirLock // lock to protect the directory and BlockMap private ReentrantReadWriteLock dirLock; @@ -157,7 +153,7 @@ boolean hasReadLock() { this.dirLock = new ReentrantReadWriteLock(true); // fair this.cond = dirLock.writeLock().newCondition(); rootDir = createRoot(ns); - inodeMap = initInodeMap(rootDir); + inodeMap = INodeMap.newInstance(rootDir); this.fsImage = fsImage; int configuredLimit = conf.getInt( DFSConfigKeys.DFS_LIST_LIMIT, DFSConfigKeys.DFS_LIST_LIMIT_DEFAULT); @@ -180,16 +176,6 @@ boolean hasReadLock() { nameCache = new NameCache(threshold); namesystem = ns; } - - private static GSet initInodeMap( - INodeDirectory rootDir) { - // Compute the map capacity by allocating 1% of total memory - int capacity = LightWeightGSet.computeCapacity(1, "INodeMap"); - GSet map - = new LightWeightGSet(capacity); - map.put(rootDir); - return map; - } private FSNamesystem getFSNamesystem() { return namesystem; @@ -608,7 +594,8 @@ boolean unprotectedRenameTo(String src, String dst, long timestamp) // snapshot is taken on the dst tree, changes will be recorded in the latest // snapshot of the src tree. if (isSrcInSnapshot) { - srcChild = srcChild.recordModification(srcIIP.getLatestSnapshot()); + srcChild = srcChild.recordModification(srcIIP.getLatestSnapshot(), + inodeMap); srcIIP.setLastINode(srcChild); } @@ -618,8 +605,9 @@ boolean unprotectedRenameTo(String src, String dst, long timestamp) int srcRefDstSnapshot = srcChildIsReference ? srcChild.asReference() .getDstSnapshotId() : Snapshot.INVALID_ID; if (isSrcInSnapshot) { - final INodeReference.WithName withName = srcIIP.getINode(-2).asDirectory() - .replaceChild4ReferenceWithName(srcChild, srcIIP.getLatestSnapshot()); + final INodeReference.WithName withName = + srcIIP.getINode(-2).asDirectory().replaceChild4ReferenceWithName( + srcChild, srcIIP.getLatestSnapshot()); withCount = (INodeReference.WithCount) withName.getReferredINode(); srcChild = withName; srcIIP.setLastINode(srcChild); @@ -676,9 +664,11 @@ boolean unprotectedRenameTo(String src, String dst, long timestamp) } // update modification time of dst and the parent of src final INode srcParent = srcIIP.getINode(-2); - srcParent.updateModificationTime(timestamp, srcIIP.getLatestSnapshot()); + srcParent.updateModificationTime(timestamp, srcIIP.getLatestSnapshot(), + inodeMap); dstParent = dstIIP.getINode(-2); // refresh dstParent - dstParent.updateModificationTime(timestamp, dstIIP.getLatestSnapshot()); + dstParent.updateModificationTime(timestamp, dstIIP.getLatestSnapshot(), + inodeMap); // update moved leases with new filename getFSNamesystem().unprotectedChangeLease(src, dst); @@ -859,7 +849,8 @@ boolean unprotectedRenameTo(String src, String dst, long timestamp, // snapshot is taken on the dst tree, changes will be recorded in the latest // snapshot of the src tree. if (isSrcInSnapshot) { - srcChild = srcChild.recordModification(srcIIP.getLatestSnapshot()); + srcChild = srcChild.recordModification(srcIIP.getLatestSnapshot(), + inodeMap); srcIIP.setLastINode(srcChild); } @@ -937,9 +928,11 @@ boolean unprotectedRenameTo(String src, String dst, long timestamp, } final INode srcParent = srcIIP.getINode(-2); - srcParent.updateModificationTime(timestamp, srcIIP.getLatestSnapshot()); + srcParent.updateModificationTime(timestamp, srcIIP.getLatestSnapshot(), + inodeMap); dstParent = dstIIP.getINode(-2); - dstParent.updateModificationTime(timestamp, dstIIP.getLatestSnapshot()); + dstParent.updateModificationTime(timestamp, dstIIP.getLatestSnapshot(), + inodeMap); // update moved lease with new filename getFSNamesystem().unprotectedChangeLease(src, dst); @@ -1071,7 +1064,8 @@ Block[] unprotectedSetReplication(String src, short replication, updateCount(iip, 0, dsDelta, true); } - file = file.setFileReplication(replication, iip.getLatestSnapshot()); + file = file.setFileReplication(replication, iip.getLatestSnapshot(), + inodeMap); final short newBR = file.getBlockReplication(); // check newBR < oldBR case. @@ -1137,7 +1131,8 @@ void unprotectedSetPermission(String src, FsPermission permissions) if (inode == null) { throw new FileNotFoundException("File does not exist: " + src); } - inode.setPermission(permissions, inodesInPath.getLatestSnapshot()); + inode.setPermission(permissions, inodesInPath.getLatestSnapshot(), + inodeMap); } void setOwner(String src, String username, String groupname) @@ -1162,10 +1157,11 @@ void unprotectedSetOwner(String src, String username, String groupname) throw new FileNotFoundException("File does not exist: " + src); } if (username != null) { - inode = inode.setUser(username, inodesInPath.getLatestSnapshot()); + inode = inode.setUser(username, inodesInPath.getLatestSnapshot(), + inodeMap); } if (groupname != null) { - inode.setGroup(groupname, inodesInPath.getLatestSnapshot()); + inode.setGroup(groupname, inodesInPath.getLatestSnapshot(), inodeMap); } } @@ -1237,12 +1233,15 @@ void unprotectedConcat(String target, String [] srcs, long timestamp) if(nodeToRemove == null) continue; nodeToRemove.setBlocks(null); - trgParent.removeChild(nodeToRemove, trgLatestSnapshot); + trgParent.removeChild(nodeToRemove, trgLatestSnapshot, null); count++; } - trgInode.setModificationTime(timestamp, trgLatestSnapshot); - trgParent.updateModificationTime(timestamp, trgLatestSnapshot); + // update inodeMap + removeFromInodeMap(Arrays.asList(allSrcInodes)); + + trgInode.setModificationTime(timestamp, trgLatestSnapshot, inodeMap); + trgParent.updateModificationTime(timestamp, trgLatestSnapshot, inodeMap); // update quota on the parent directory ('count' files removed, 0 space) unprotectedUpdateCount(trgIIP, trgINodes.length-1, -count, 0); } @@ -1384,7 +1383,7 @@ long unprotectedDelete(INodesInPath iip, BlocksMapUpdateInfo collectedBlocks, // record modification final Snapshot latestSnapshot = iip.getLatestSnapshot(); - targetNode = targetNode.recordModification(latestSnapshot); + targetNode = targetNode.recordModification(latestSnapshot, inodeMap); iip.setLastINode(targetNode); // Remove the node from the namespace @@ -1395,7 +1394,7 @@ long unprotectedDelete(INodesInPath iip, BlocksMapUpdateInfo collectedBlocks, // set the parent's modification time final INodeDirectory parent = targetNode.getParent(); - parent.updateModificationTime(mtime, latestSnapshot); + parent.updateModificationTime(mtime, latestSnapshot, inodeMap); if (removed == 0) { return 0; } @@ -1466,8 +1465,7 @@ void unprotectedReplaceINodeFile(final String path, final INodeFile oldnode, final INodeFile newnode) { Preconditions.checkState(hasWriteLock()); - oldnode.getParent().replaceChild(oldnode, newnode); - addToInodeMapUnprotected(newnode); + oldnode.getParent().replaceChild(oldnode, newnode, inodeMap); oldnode.clear(); /* Currently oldnode and newnode are assumed to contain the same @@ -1984,15 +1982,6 @@ private void unprotectedMkdir(long inodeId, INodesInPath inodesInPath, } } - private INode getFromINodeMap(INode inode) { - readLock(); - try { - return inodeMap.get(inode); - } finally { - readUnlock(); - } - } - /** * Add the given child to the namespace. * @param src The full path name of the child node. @@ -2194,14 +2183,14 @@ private boolean addChild(INodesInPath iip, int pos, updateCount(iip, pos, counts.get(Quota.NAMESPACE), counts.get(Quota.DISKSPACE), checkQuota); final INodeDirectory parent = inodes[pos-1].asDirectory(); - final boolean added = parent.addChild(child, true, iip.getLatestSnapshot()); + final boolean added = parent.addChild(child, true, iip.getLatestSnapshot(), + inodeMap); if (!added) { updateCountNoQuotaCheck(iip, pos, -counts.get(Quota.NAMESPACE), -counts.get(Quota.DISKSPACE)); } else { - // update parent node iip.setINode(pos - 1, child.getParent()); - addToInodeMapUnprotected(child); + addToInodeMap(child); } return added; } @@ -2228,13 +2217,12 @@ private long removeLastINode(final INodesInPath iip) final Snapshot latestSnapshot = iip.getLatestSnapshot(); final INode last = iip.getLastINode(); final INodeDirectory parent = iip.getINode(-2).asDirectory(); - if (!parent.removeChild(last, latestSnapshot)) { + if (!parent.removeChild(last, latestSnapshot, inodeMap)) { return -1; } - if (parent != last.getParent()) { - // parent is changed - addToInodeMapUnprotected(last.getParent()); - iip.setINode(-2, last.getParent()); + INodeDirectory newParent = last.getParent(); + if (parent != newParent) { + iip.setINode(-2, newParent); } if (!last.isInLatestSnapshot(latestSnapshot)) { @@ -2277,22 +2265,51 @@ ContentSummary getContentSummary(String src) } } - /** This method is always called with writeLock held */ - final void addToInodeMapUnprotected(INode inode) { + public INodeMap getINodeMap() { + return inodeMap; + } + + /** + * This method is always called with writeLock of FSDirectory held. + */ + public final void addToInodeMap(INode inode) { if (inode instanceof INodeWithAdditionalFields) { inodeMap.put((INodeWithAdditionalFields)inode); } } - /* This method is always called with writeLock held */ - final void removeFromInodeMap(List inodes) { + /** + * This method is always called with writeLock of FSDirectory held. + */ + public final void removeFromInodeMap(List inodes) { if (inodes != null) { for (INode inode : inodes) { - inodeMap.remove(inode); + if (inode != null && inode instanceof INodeWithAdditionalFields) { + inodeMap.remove(inode); + } } } } + /** + * Get the inode from inodeMap based on its inode id. + * @param id The given id + * @return The inode associated with the given id + */ + public INode getInode(long id) { + readLock(); + try { + return inodeMap.get(id); + } finally { + readUnlock(); + } + } + + @VisibleForTesting + int getInodeMapSize() { + return inodeMap.size(); + } + /** * See {@link ClientProtocol#setQuota(String, long, long)} for the contract. * Sets quota for for a directory. @@ -2354,18 +2371,12 @@ INodeDirectory unprotectedSetQuota(String src, long nsQuota, long dsQuota) if (!(quotaNode instanceof INodeDirectoryWithSnapshot)) { // will not come here for root because root is snapshottable and // root's nsQuota is always set - INodeDirectory newNode = quotaNode.replaceSelf4INodeDirectory(); - // update the inodeMap - inodeMap.put(newNode); - return newNode; - } + return quotaNode.replaceSelf4INodeDirectory(inodeMap); + } } } else { // a non-quota directory; so replace it with a directory with quota - INodeDirectory newNode = dirNode.replaceSelf4Quota(latest, nsQuota, dsQuota); - // update the inodeMap - inodeMap.put(newNode); - return newNode; + return dirNode.replaceSelf4Quota(latest, nsQuota, dsQuota, inodeMap); } return (oldNsQuota != nsQuota || oldDsQuota != dsQuota) ? dirNode : null; } @@ -2431,7 +2442,7 @@ private boolean unprotectedSetTimes(INode inode, long mtime, assert hasWriteLock(); boolean status = false; if (mtime != -1) { - inode = inode.setModificationTime(mtime, latest); + inode = inode.setModificationTime(mtime, latest, inodeMap); status = true; } if (atime != -1) { @@ -2442,7 +2453,7 @@ private boolean unprotectedSetTimes(INode inode, long mtime, if (atime <= inodeTime + getFSNamesystem().getAccessTimePrecision() && !force) { status = false; } else { - inode.setAccessTime(atime, latest); + inode.setAccessTime(atime, latest, inodeMap); status = true; } } @@ -2458,7 +2469,7 @@ void reset() { setReady(false); rootDir = createRoot(getFSNamesystem()); inodeMap.clear(); - addToInodeMapUnprotected(rootDir); + addToInodeMap(rootDir); nameCache.reset(); } finally { writeUnlock(); @@ -2622,49 +2633,6 @@ void cacheName(INode inode) { void shutdown() { nameCache.reset(); inodeMap.clear(); - inodeMap = null; - } - - INode getInode(long id) { - INode inode = new INodeWithAdditionalFields(id, null, new PermissionStatus( - "", "", new FsPermission((short) 0)), 0, 0) { - - @Override - INode recordModification(Snapshot latest) throws QuotaExceededException { - return null; - } - - @Override - public void destroyAndCollectBlocks(BlocksMapUpdateInfo collectedBlocks, - List removedINodes) { - // Nothing to do - } - - @Override - public Counts computeQuotaUsage(Counts counts, boolean useCache, - int lastSnapshotId) { - return null; - } - - @Override - public Content.Counts computeContentSummary(Content.Counts counts) { - return null; - } - - @Override - public CountsMap computeContentSummary(CountsMap countsMap) { - return null; - } - - @Override - public Counts cleanSubtree(Snapshot snapshot, Snapshot prior, - BlocksMapUpdateInfo collectedBlocks, List removedINodes) - throws QuotaExceededException { - return null; - } - }; - - return getFromINodeMap(inode); } /** @@ -2732,11 +2700,6 @@ static String resolvePath(String src, byte[][] pathComponents, FSDirectory fsd) return path.toString(); } - @VisibleForTesting - int getInodeMapSize() { - return inodeMap.size(); - } - /** Check if a given inode name is reserved */ public static boolean isReservedName(INode inode) { return CHECK_RESERVED_FILE_NAMES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java index 04273e2ba91..17b42f4cc34 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java @@ -315,8 +315,8 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir, // update the block list. // Update the salient file attributes. - newFile.setAccessTime(addCloseOp.atime, null); - newFile.setModificationTime(addCloseOp.mtime, null); + newFile.setAccessTime(addCloseOp.atime, null, fsDir.getINodeMap()); + newFile.setModificationTime(addCloseOp.mtime, null, fsDir.getINodeMap()); updateBlocks(fsDir, addCloseOp, newFile); break; } @@ -334,8 +334,8 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir, final INodeFile oldFile = INodeFile.valueOf(iip.getINode(0), addCloseOp.path); // Update the salient file attributes. - oldFile.setAccessTime(addCloseOp.atime, null); - oldFile.setModificationTime(addCloseOp.mtime, null); + oldFile.setAccessTime(addCloseOp.atime, null, fsDir.getINodeMap()); + oldFile.setModificationTime(addCloseOp.mtime, null, fsDir.getINodeMap()); updateBlocks(fsDir, addCloseOp, oldFile); // Now close the file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java index a6b5eb704c2..6fad2243493 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java @@ -560,7 +560,7 @@ public INode loadINodeWithLocalName(boolean isSnapshotINode, final byte[] localName = FSImageSerialization.readLocalName(in); INode inode = loadINode(localName, isSnapshotINode, in); if (LayoutVersion.supports(Feature.ADD_INODE_ID, getLayoutVersion())) { - namesystem.dir.addToInodeMapUnprotected(inode); + namesystem.dir.addToInodeMap(inode); } return inode; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 9e6a9ae27ad..7f6d49adc0a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -1990,7 +1990,7 @@ private LocatedBlock startFileInternal(FSPermissionChecker pc, String src, LocatedBlock prepareFileForWrite(String src, INodeFile file, String leaseHolder, String clientMachine, DatanodeDescriptor clientNode, boolean writeToEditLog, Snapshot latestSnapshot) throws IOException { - file = file.recordModification(latestSnapshot); + file = file.recordModification(latestSnapshot, dir.getINodeMap()); final INodeFileUnderConstruction cons = file.toUnderConstruction( leaseHolder, clientMachine, clientNode); @@ -3400,7 +3400,8 @@ private void finalizeINodeFileUnderConstruction(String src, assert hasWriteLock(); leaseManager.removeLease(pendingFile.getClientName(), src); - pendingFile = pendingFile.recordModification(latestSnapshot); + pendingFile = pendingFile.recordModification(latestSnapshot, + dir.getINodeMap()); // The file is no longer pending. // Create permanent INode, update blocks @@ -5823,7 +5824,12 @@ void allowSnapshot(String path) throws SafeModeException, IOException { } checkSuperuserPrivilege(); - snapshotManager.setSnapshottable(path); + dir.writeLock(); + try { + snapshotManager.setSnapshottable(path); + } finally { + dir.writeUnlock(); + } getEditLog().logAllowSnapshot(path); } finally { writeUnlock(); @@ -5846,7 +5852,12 @@ void disallowSnapshot(String path) throws SafeModeException, IOException { } checkSuperuserPrivilege(); - snapshotManager.resetSnapshottable(path); + dir.writeLock(); + try { + snapshotManager.resetSnapshottable(path); + } finally { + dir.writeUnlock(); + } getEditLog().logDisallowSnapshot(path); } finally { writeUnlock(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java index 8b223fe6de5..57fa7fbdcbe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java @@ -96,9 +96,9 @@ public final String getUserName() { abstract void setUser(String user); /** Set user */ - final INode setUser(String user, Snapshot latest) + final INode setUser(String user, Snapshot latest, INodeMap inodeMap) throws QuotaExceededException { - final INode nodeToUpdate = recordModification(latest); + final INode nodeToUpdate = recordModification(latest, inodeMap); nodeToUpdate.setUser(user); return nodeToUpdate; } @@ -119,9 +119,9 @@ public final String getGroupName() { abstract void setGroup(String group); /** Set group */ - final INode setGroup(String group, Snapshot latest) + final INode setGroup(String group, Snapshot latest, INodeMap inodeMap) throws QuotaExceededException { - final INode nodeToUpdate = recordModification(latest); + final INode nodeToUpdate = recordModification(latest, inodeMap); nodeToUpdate.setGroup(group); return nodeToUpdate; } @@ -143,9 +143,9 @@ public final FsPermission getFsPermission() { abstract void setPermission(FsPermission permission); /** Set the {@link FsPermission} of this {@link INode} */ - INode setPermission(FsPermission permission, Snapshot latest) - throws QuotaExceededException { - final INode nodeToUpdate = recordModification(latest); + INode setPermission(FsPermission permission, Snapshot latest, + INodeMap inodeMap) throws QuotaExceededException { + final INode nodeToUpdate = recordModification(latest, inodeMap); nodeToUpdate.setPermission(permission); return nodeToUpdate; } @@ -219,12 +219,14 @@ public final boolean shouldRecordInSrcSnapshot(final Snapshot latestInDst) { * * @param latest the latest snapshot that has been taken. * Note that it is null if no snapshots have been taken. + * @param inodeMap while recording modification, the inode or its parent may + * get replaced, and the inodeMap needs to be updated. * @return The current inode, which usually is the same object of this inode. * However, in some cases, this inode may be replaced with a new inode * for maintaining snapshots. The current inode is then the new inode. */ - abstract INode recordModification(final Snapshot latest) - throws QuotaExceededException; + abstract INode recordModification(final Snapshot latest, + final INodeMap inodeMap) throws QuotaExceededException; /** Check whether it's a reference. */ public boolean isReference() { @@ -564,16 +566,16 @@ public final long getModificationTime() { } /** Update modification time if it is larger than the current value. */ - public abstract INode updateModificationTime(long mtime, Snapshot latest) - throws QuotaExceededException; + public abstract INode updateModificationTime(long mtime, Snapshot latest, + INodeMap inodeMap) throws QuotaExceededException; /** Set the last modification time of inode. */ public abstract void setModificationTime(long modificationTime); /** Set the last modification time of inode. */ - public final INode setModificationTime(long modificationTime, Snapshot latest) - throws QuotaExceededException { - final INode nodeToUpdate = recordModification(latest); + public final INode setModificationTime(long modificationTime, + Snapshot latest, INodeMap inodeMap) throws QuotaExceededException { + final INode nodeToUpdate = recordModification(latest, inodeMap); nodeToUpdate.setModificationTime(modificationTime); return nodeToUpdate; } @@ -599,9 +601,9 @@ public final long getAccessTime() { /** * Set last access time of inode. */ - public final INode setAccessTime(long accessTime, Snapshot latest) - throws QuotaExceededException { - final INode nodeToUpdate = recordModification(latest); + public final INode setAccessTime(long accessTime, Snapshot latest, + INodeMap inodeMap) throws QuotaExceededException { + final INode nodeToUpdate = recordModification(latest, inodeMap); nodeToUpdate.setAccessTime(accessTime); return nodeToUpdate; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java index 48add72785f..afaf98c0266 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java @@ -113,11 +113,11 @@ private int searchChildren(byte[] name) { * @param child the child inode to be removed * @param latest See {@link INode#recordModification(Snapshot)}. */ - public boolean removeChild(INode child, Snapshot latest) - throws QuotaExceededException { + public boolean removeChild(INode child, Snapshot latest, + final INodeMap inodeMap) throws QuotaExceededException { if (isInLatestSnapshot(latest)) { - return replaceSelf4INodeDirectoryWithSnapshot() - .removeChild(child, latest); + return replaceSelf4INodeDirectoryWithSnapshot(inodeMap) + .removeChild(child, latest, inodeMap); } return removeChild(child); @@ -147,59 +147,66 @@ protected final boolean removeChild(final INode child) { * {@link INodeDirectoryWithSnapshot} depending on the latest snapshot. */ INodeDirectoryWithQuota replaceSelf4Quota(final Snapshot latest, - final long nsQuota, final long dsQuota) throws QuotaExceededException { + final long nsQuota, final long dsQuota, final INodeMap inodeMap) + throws QuotaExceededException { Preconditions.checkState(!(this instanceof INodeDirectoryWithQuota), "this is already an INodeDirectoryWithQuota, this=%s", this); if (!this.isInLatestSnapshot(latest)) { final INodeDirectoryWithQuota q = new INodeDirectoryWithQuota( this, true, nsQuota, dsQuota); - replaceSelf(q); + replaceSelf(q, inodeMap); return q; } else { final INodeDirectoryWithSnapshot s = new INodeDirectoryWithSnapshot(this); s.setQuota(nsQuota, dsQuota); - return replaceSelf(s).saveSelf2Snapshot(latest, this); + return replaceSelf(s, inodeMap).saveSelf2Snapshot(latest, this); } } /** Replace itself with an {@link INodeDirectorySnapshottable}. */ public INodeDirectorySnapshottable replaceSelf4INodeDirectorySnapshottable( - Snapshot latest) throws QuotaExceededException { + Snapshot latest, final INodeMap inodeMap) throws QuotaExceededException { Preconditions.checkState(!(this instanceof INodeDirectorySnapshottable), "this is already an INodeDirectorySnapshottable, this=%s", this); final INodeDirectorySnapshottable s = new INodeDirectorySnapshottable(this); - replaceSelf(s).saveSelf2Snapshot(latest, this); + replaceSelf(s, inodeMap).saveSelf2Snapshot(latest, this); return s; } /** Replace itself with an {@link INodeDirectoryWithSnapshot}. */ - public INodeDirectoryWithSnapshot replaceSelf4INodeDirectoryWithSnapshot() { - return replaceSelf(new INodeDirectoryWithSnapshot(this)); + public INodeDirectoryWithSnapshot replaceSelf4INodeDirectoryWithSnapshot( + final INodeMap inodeMap) { + return replaceSelf(new INodeDirectoryWithSnapshot(this), inodeMap); } /** Replace itself with {@link INodeDirectory}. */ - public INodeDirectory replaceSelf4INodeDirectory() { + public INodeDirectory replaceSelf4INodeDirectory(final INodeMap inodeMap) { Preconditions.checkState(getClass() != INodeDirectory.class, "the class is already INodeDirectory, this=%s", this); - return replaceSelf(new INodeDirectory(this, true)); + return replaceSelf(new INodeDirectory(this, true), inodeMap); } /** Replace itself with the given directory. */ - private final N replaceSelf(final N newDir) { + private final N replaceSelf(final N newDir, + final INodeMap inodeMap) { final INodeReference ref = getParentReference(); if (ref != null) { ref.setReferredINode(newDir); + if (inodeMap != null) { + inodeMap.put(newDir); + } } else { final INodeDirectory parent = getParent(); Preconditions.checkArgument(parent != null, "parent is null, this=%s", this); - parent.replaceChild(this, newDir); + parent.replaceChild(this, newDir, inodeMap); } clear(); return newDir; } /** Replace the given child with a new child. */ - public void replaceChild(INode oldChild, final INode newChild) { + public void replaceChild(INode oldChild, final INode newChild, + final INodeMap inodeMap) { Preconditions.checkNotNull(children); final int i = searchChildren(newChild.getLocalNameBytes()); Preconditions.checkState(i >= 0); @@ -220,9 +227,12 @@ public void replaceChild(INode oldChild, final INode newChild) { (WithCount) oldChild.asReference().getReferredINode(); withCount.removeReference(oldChild.asReference()); } - // do the replacement children.set(i, newChild); } + // update the inodeMap + if (inodeMap != null) { + inodeMap.put(newChild); + } } INodeReference.WithName replaceChild4ReferenceWithName(INode oldChild, @@ -241,43 +251,47 @@ INodeReference.WithName replaceChild4ReferenceWithName(INode oldChild, } final INodeReference.WithName ref = new INodeReference.WithName(this, withCount, oldChild.getLocalNameBytes(), latest.getId()); - replaceChild(oldChild, ref); + replaceChild(oldChild, ref, null); return ref; } - private void replaceChildFile(final INodeFile oldChild, final INodeFile newChild) { - replaceChild(oldChild, newChild); + private void replaceChildFile(final INodeFile oldChild, + final INodeFile newChild, final INodeMap inodeMap) { + replaceChild(oldChild, newChild, inodeMap); oldChild.clear(); newChild.updateBlockCollection(); } /** Replace a child {@link INodeFile} with an {@link INodeFileWithSnapshot}. */ INodeFileWithSnapshot replaceChild4INodeFileWithSnapshot( - final INodeFile child) { + final INodeFile child, final INodeMap inodeMap) { Preconditions.checkArgument(!(child instanceof INodeFileWithSnapshot), "Child file is already an INodeFileWithSnapshot, child=" + child); final INodeFileWithSnapshot newChild = new INodeFileWithSnapshot(child); - replaceChildFile(child, newChild); + replaceChildFile(child, newChild, inodeMap); return newChild; } /** Replace a child {@link INodeFile} with an {@link INodeFileUnderConstructionWithSnapshot}. */ INodeFileUnderConstructionWithSnapshot replaceChild4INodeFileUcWithSnapshot( - final INodeFileUnderConstruction child) { + final INodeFileUnderConstruction child, final INodeMap inodeMap) { Preconditions.checkArgument(!(child instanceof INodeFileUnderConstructionWithSnapshot), "Child file is already an INodeFileUnderConstructionWithSnapshot, child=" + child); final INodeFileUnderConstructionWithSnapshot newChild = new INodeFileUnderConstructionWithSnapshot(child, null); - replaceChildFile(child, newChild); + replaceChildFile(child, newChild, inodeMap); return newChild; } @Override - public INodeDirectory recordModification(Snapshot latest) - throws QuotaExceededException { - return isInLatestSnapshot(latest)? - replaceSelf4INodeDirectoryWithSnapshot().recordModification(latest) - : this; + public INodeDirectory recordModification(Snapshot latest, + final INodeMap inodeMap) throws QuotaExceededException { + if (isInLatestSnapshot(latest)) { + return replaceSelf4INodeDirectoryWithSnapshot(inodeMap) + .recordModification(latest, inodeMap); + } else { + return this; + } } /** @@ -286,12 +300,13 @@ public INodeDirectory recordModification(Snapshot latest) * @return the child inode, which may be replaced. */ public INode saveChild2Snapshot(final INode child, final Snapshot latest, - final INode snapshotCopy) throws QuotaExceededException { + final INode snapshotCopy, final INodeMap inodeMap) + throws QuotaExceededException { if (latest == null) { return child; } - return replaceSelf4INodeDirectoryWithSnapshot() - .saveChild2Snapshot(child, latest, snapshotCopy); + return replaceSelf4INodeDirectoryWithSnapshot(inodeMap) + .saveChild2Snapshot(child, latest, snapshotCopy, inodeMap); } /** @@ -378,24 +393,28 @@ static int nextChild(ReadOnlyList children, byte[] name) { * @param setModTime set modification time for the parent node * not needed when replaying the addition and * the parent already has the proper mod time + * @param inodeMap update the inodeMap if the directory node gets replaced * @return false if the child with this name already exists; * otherwise, return true; */ public boolean addChild(INode node, final boolean setModTime, - final Snapshot latest) throws QuotaExceededException { + final Snapshot latest, final INodeMap inodeMap) + throws QuotaExceededException { final int low = searchChildren(node.getLocalNameBytes()); if (low >= 0) { return false; } if (isInLatestSnapshot(latest)) { - return replaceSelf4INodeDirectoryWithSnapshot() - .addChild(node, setModTime, latest); + INodeDirectoryWithSnapshot sdir = + replaceSelf4INodeDirectoryWithSnapshot(inodeMap); + boolean added = sdir.addChild(node, setModTime, latest, inodeMap); + return added; } addChild(node, low); if (setModTime) { // update modification time of the parent directory - updateModificationTime(node.getModificationTime(), latest); + updateModificationTime(node.getModificationTime(), latest, inodeMap); } return true; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java index e3e1a000deb..655438ee45c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hdfs.server.namenode.snapshot.FileWithSnapshot.FileDiff; import org.apache.hadoop.hdfs.server.namenode.snapshot.FileDiffList; import org.apache.hadoop.hdfs.server.namenode.snapshot.FileWithSnapshot.Util; +import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeFileWithSnapshot; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; import com.google.common.annotations.VisibleForTesting; @@ -155,12 +156,16 @@ public INodeFile getSnapshotINode(final Snapshot snapshot) { } @Override - public INodeFile recordModification(final Snapshot latest) - throws QuotaExceededException { - return isInLatestSnapshot(latest)? - getParent().replaceChild4INodeFileWithSnapshot(this) - .recordModification(latest) - : this; + public INodeFile recordModification(final Snapshot latest, + final INodeMap inodeMap) throws QuotaExceededException { + if (isInLatestSnapshot(latest)) { + INodeFileWithSnapshot newFile = getParent() + .replaceChild4INodeFileWithSnapshot(this, inodeMap) + .recordModification(latest, inodeMap); + return newFile; + } else { + return this; + } } /** @@ -179,9 +184,9 @@ final void setPermission(FsPermission permission) { * the {@link FsAction#EXECUTE} action, if any, is ignored. */ @Override - final INode setPermission(FsPermission permission, Snapshot latest) - throws QuotaExceededException { - return super.setPermission(permission.applyUMask(UMASK), latest); + final INode setPermission(FsPermission permission, Snapshot latest, + final INodeMap inodeMap) throws QuotaExceededException { + return super.setPermission(permission.applyUMask(UMASK), latest, inodeMap); } /** @return the replication factor of the file. */ @@ -211,9 +216,9 @@ public final void setFileReplication(short replication) { } /** Set the replication factor of this file. */ - public final INodeFile setFileReplication(short replication, Snapshot latest) - throws QuotaExceededException { - final INodeFile nodeToUpdate = recordModification(latest); + public final INodeFile setFileReplication(short replication, Snapshot latest, + final INodeMap inodeMap) throws QuotaExceededException { + final INodeFile nodeToUpdate = recordModification(latest, inodeMap); nodeToUpdate.setFileReplication(replication); return nodeToUpdate; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java index 6919b47fed4..bcc6e8187e9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileUnderConstruction.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.blockmanagement.MutableBlockCollection; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState; +import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeFileUnderConstructionWithSnapshot; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; import com.google.common.base.Preconditions; @@ -130,12 +131,16 @@ protected INodeFile toINodeFile(long mtime) { } @Override - public INodeFileUnderConstruction recordModification(final Snapshot latest) - throws QuotaExceededException { - return isInLatestSnapshot(latest)? - getParent().replaceChild4INodeFileUcWithSnapshot(this) - .recordModification(latest) - : this; + public INodeFileUnderConstruction recordModification(final Snapshot latest, + final INodeMap inodeMap) throws QuotaExceededException { + if (isInLatestSnapshot(latest)) { + INodeFileUnderConstructionWithSnapshot newFile = getParent() + .replaceChild4INodeFileUcWithSnapshot(this, inodeMap) + .recordModification(latest, inodeMap); + return newFile; + } else { + return this; + } } /** Assert all blocks are complete. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java new file mode 100644 index 00000000000..e0e4cdf49ee --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java @@ -0,0 +1,137 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode; + +import java.util.List; + +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.fs.permission.PermissionStatus; +import org.apache.hadoop.hdfs.protocol.QuotaExceededException; +import org.apache.hadoop.hdfs.server.namenode.Content.CountsMap; +import org.apache.hadoop.hdfs.server.namenode.Quota.Counts; +import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; +import org.apache.hadoop.hdfs.util.GSet; +import org.apache.hadoop.hdfs.util.LightWeightGSet; + +import com.google.common.base.Preconditions; + +/** + * Storing all the {@link INode}s and maintaining the mapping between INode ID + * and INode. + */ +public class INodeMap { + + static INodeMap newInstance(INodeDirectory rootDir) { + // Compute the map capacity by allocating 1% of total memory + int capacity = LightWeightGSet.computeCapacity(1, "INodeMap"); + GSet map + = new LightWeightGSet(capacity); + map.put(rootDir); + return new INodeMap(map); + } + + /** Synchronized by external lock. */ + private GSet map; + + private INodeMap(GSet map) { + Preconditions.checkArgument(map != null); + this.map = map; + } + + /** + * Add an {@link INode} into the {@link INode} map. Replace the old value if + * necessary. + * @param inode The {@link INode} to be added to the map. + */ + public final void put(INode inode) { + if (inode instanceof INodeWithAdditionalFields) { + map.put((INodeWithAdditionalFields)inode); + } + } + + /** + * Remove a {@link INode} from the map. + * @param inode The {@link INode} to be removed. + */ + public final void remove(INode inode) { + map.remove(inode); + } + + /** + * @return The size of the map. + */ + public int size() { + return map.size(); + } + + /** + * Get the {@link INode} with the given id from the map. + * @param id ID of the {@link INode}. + * @return The {@link INode} in the map with the given id. Return null if no + * such {@link INode} in the map. + */ + public INode get(long id) { + INode inode = new INodeWithAdditionalFields(id, null, new PermissionStatus( + "", "", new FsPermission((short) 0)), 0, 0) { + + @Override + INode recordModification(Snapshot latest, INodeMap inodeMap) + throws QuotaExceededException { + return null; + } + + @Override + public void destroyAndCollectBlocks(BlocksMapUpdateInfo collectedBlocks, + List removedINodes) { + // Nothing to do + } + + @Override + public Counts computeQuotaUsage(Counts counts, boolean useCache, + int lastSnapshotId) { + return null; + } + + @Override + public Content.Counts computeContentSummary(Content.Counts counts) { + return null; + } + + @Override + public CountsMap computeContentSummary(CountsMap countsMap) { + return null; + } + + @Override + public Counts cleanSubtree(Snapshot snapshot, Snapshot prior, + BlocksMapUpdateInfo collectedBlocks, List removedINodes) + throws QuotaExceededException { + return null; + } + }; + + return map.get(inode); + } + + /** + * Clear the {@link #map} + */ + public void clear() { + map.clear(); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java index 7735d8ac153..f9f03b5827f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java @@ -197,9 +197,9 @@ public final long getModificationTime(Snapshot snapshot) { } @Override - public final INode updateModificationTime(long mtime, Snapshot latest) - throws QuotaExceededException { - return referred.updateModificationTime(mtime, latest); + public final INode updateModificationTime(long mtime, Snapshot latest, + INodeMap inodeMap) throws QuotaExceededException { + return referred.updateModificationTime(mtime, latest, inodeMap); } @Override @@ -218,8 +218,9 @@ public final void setAccessTime(long accessTime) { } @Override - final INode recordModification(Snapshot latest) throws QuotaExceededException { - referred.recordModification(latest); + final INode recordModification(Snapshot latest, final INodeMap inodeMap) + throws QuotaExceededException { + referred.recordModification(latest, inodeMap); // reference is never replaced return this; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java index aee1533db40..96873fa5803 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java @@ -46,10 +46,13 @@ public class INodeSymlink extends INodeWithAdditionalFields { } @Override - INode recordModification(Snapshot latest) throws QuotaExceededException { - return isInLatestSnapshot(latest)? - getParent().saveChild2Snapshot(this, latest, new INodeSymlink(this)) - : this; + INode recordModification(Snapshot latest, final INodeMap inodeMap) + throws QuotaExceededException { + if (isInLatestSnapshot(latest)) { + INodeDirectory parent = getParent(); + parent.saveChild2Snapshot(this, latest, new INodeSymlink(this), inodeMap); + } + return this; } /** @return true unconditionally. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java index aa78d9e7ea7..99d30582221 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java @@ -221,13 +221,13 @@ final long getModificationTime(Snapshot snapshot) { /** Update modification time if it is larger than the current value. */ - public final INode updateModificationTime(long mtime, Snapshot latest) - throws QuotaExceededException { + public final INode updateModificationTime(long mtime, Snapshot latest, + final INodeMap inodeMap) throws QuotaExceededException { Preconditions.checkState(isDirectory()); if (mtime <= modificationTime) { return this; } - return setModificationTime(mtime, latest); + return setModificationTime(mtime, latest, inodeMap); } final void cloneModificationTime(INodeWithAdditionalFields that) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java index 4cd8764dbea..a1d77687071 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodesInPath.java @@ -28,7 +28,6 @@ import org.apache.hadoop.hdfs.protocol.UnresolvedPathException; import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectorySnapshottable; import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeDirectoryWithSnapshot; -import org.apache.hadoop.hdfs.server.namenode.snapshot.INodeFileWithSnapshot; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; import com.google.common.base.Preconditions; @@ -160,10 +159,6 @@ static INodesInPath resolve(final INodeDirectory startingDir, && curNode.asDirectory() instanceof INodeDirectoryWithSnapshot) { lastSnapshot = ((INodeDirectoryWithSnapshot) curNode .asDirectory()).getLastSnapshot(); - } else if (curNode.isFile() - && curNode.asFile() instanceof INodeFileWithSnapshot) { - lastSnapshot = ((INodeFileWithSnapshot) curNode - .asFile()).getDiffs().getLastSnapshot(); } existing.setSnapshot(lastSnapshot); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java index 452b2380016..8db9a93c306 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectorySnapshottable.java @@ -41,6 +41,7 @@ import org.apache.hadoop.hdfs.server.namenode.INode; import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; import org.apache.hadoop.hdfs.server.namenode.INodeFile; +import org.apache.hadoop.hdfs.server.namenode.INodeMap; import org.apache.hadoop.hdfs.server.namenode.Quota; import org.apache.hadoop.hdfs.util.Diff.ListType; import org.apache.hadoop.hdfs.util.ReadOnlyList; @@ -274,8 +275,8 @@ void addSnapshot(Snapshot snapshot) { } /** Add a snapshot. */ - Snapshot addSnapshot(int id, String name) - throws SnapshotException, QuotaExceededException { + Snapshot addSnapshot(int id, String name) throws SnapshotException, + QuotaExceededException { //check snapshot quota final int n = getNumSnapshots(); if (n + 1 > snapshotQuota) { @@ -296,8 +297,8 @@ Snapshot addSnapshot(int id, String name) snapshotsByNames.add(-i - 1, s); //set modification time - updateModificationTime(Time.now(), null); - s.getRoot().setModificationTime(getModificationTime(), null); + updateModificationTime(Time.now(), null, null); + s.getRoot().setModificationTime(getModificationTime(), null, null); return s; } @@ -455,13 +456,15 @@ private void computeDiffRecursively(INode node, List parentPath, * Replace itself with {@link INodeDirectoryWithSnapshot} or * {@link INodeDirectory} depending on the latest snapshot. */ - void replaceSelf(final Snapshot latest) throws QuotaExceededException { + INodeDirectory replaceSelf(final Snapshot latest, final INodeMap inodeMap) + throws QuotaExceededException { if (latest == null) { Preconditions.checkState(getLastSnapshot() == null, "latest == null but getLastSnapshot() != null, this=%s", this); - replaceSelf4INodeDirectory(); + return replaceSelf4INodeDirectory(inodeMap); } else { - replaceSelf4INodeDirectoryWithSnapshot().recordModification(latest); + return replaceSelf4INodeDirectoryWithSnapshot(inodeMap) + .recordModification(latest, null); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectoryWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectoryWithSnapshot.java index 139499a696d..33a6a9c18a7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectoryWithSnapshot.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeDirectoryWithSnapshot.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hdfs.server.namenode.INode; import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; import org.apache.hadoop.hdfs.server.namenode.INodeDirectoryWithQuota; +import org.apache.hadoop.hdfs.server.namenode.INodeMap; import org.apache.hadoop.hdfs.server.namenode.INodeReference; import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithCount; import org.apache.hadoop.hdfs.server.namenode.Quota; @@ -634,8 +635,8 @@ public INodeDirectory getSnapshotINode(Snapshot snapshot) { } @Override - public INodeDirectoryWithSnapshot recordModification(final Snapshot latest) - throws QuotaExceededException { + public INodeDirectoryWithSnapshot recordModification(final Snapshot latest, + final INodeMap inodeMap) throws QuotaExceededException { if (isInLatestSnapshot(latest) && !shouldRecordInSrcSnapshot(latest)) { return saveSelf2Snapshot(latest, null); } @@ -652,7 +653,8 @@ public INodeDirectoryWithSnapshot saveSelf2Snapshot( @Override public INode saveChild2Snapshot(final INode child, final Snapshot latest, - final INode snapshotCopy) throws QuotaExceededException { + final INode snapshotCopy, final INodeMap inodeMap) + throws QuotaExceededException { Preconditions.checkArgument(!child.isDirectory(), "child is a directory, child=%s", child); if (latest == null) { @@ -670,15 +672,15 @@ public INode saveChild2Snapshot(final INode child, final Snapshot latest, } @Override - public boolean addChild(INode inode, boolean setModTime, Snapshot latest) - throws QuotaExceededException { + public boolean addChild(INode inode, boolean setModTime, Snapshot latest, + final INodeMap inodeMap) throws QuotaExceededException { ChildrenDiff diff = null; Integer undoInfo = null; if (latest != null) { diff = diffs.checkAndAddLatestSnapshotDiff(latest, this).diff; undoInfo = diff.create(inode); } - final boolean added = super.addChild(inode, setModTime, null); + final boolean added = super.addChild(inode, setModTime, null, inodeMap); if (!added && undoInfo != null) { diff.undoCreate(inode, undoInfo); } @@ -686,8 +688,8 @@ public boolean addChild(INode inode, boolean setModTime, Snapshot latest) } @Override - public boolean removeChild(INode child, Snapshot latest) - throws QuotaExceededException { + public boolean removeChild(INode child, Snapshot latest, + final INodeMap inodeMap) throws QuotaExceededException { ChildrenDiff diff = null; UndoInfo undoInfo = null; if (latest != null) { @@ -705,8 +707,9 @@ public boolean removeChild(INode child, Snapshot latest) } @Override - public void replaceChild(final INode oldChild, final INode newChild) { - super.replaceChild(oldChild, newChild); + public void replaceChild(final INode oldChild, final INode newChild, + final INodeMap inodeMap) { + super.replaceChild(oldChild, newChild, inodeMap); diffs.replaceChild(ListType.CREATED, oldChild, newChild); } @@ -747,7 +750,9 @@ public void undoRename4ScrParent(final INodeReference oldChild, throws QuotaExceededException { diffs.removeChild(ListType.DELETED, oldChild); diffs.replaceChild(ListType.CREATED, oldChild, newChild); - addChild(newChild, true, null); + // pass null for inodeMap since the parent node will not get replaced when + // undoing rename + addChild(newChild, true, null, null); } /** @@ -759,8 +764,10 @@ public void undoRename4DstParent(final INode deletedChild, Snapshot latestSnapshot) throws QuotaExceededException { boolean removeDeletedChild = diffs.removeChild(ListType.DELETED, deletedChild); + // pass null for inodeMap since the parent node will not get replaced when + // undoing rename final boolean added = addChild(deletedChild, true, removeDeletedChild ? null - : latestSnapshot); + : latestSnapshot, null); // update quota usage if adding is successfully and the old child has not // been stored in deleted list before if (added && !removeDeletedChild) { @@ -804,7 +811,7 @@ public Quota.Counts cleanSubtree(final Snapshot snapshot, Snapshot prior, throws QuotaExceededException { Quota.Counts counts = Quota.Counts.newInstance(); if (snapshot == null) { // delete the current directory - recordModification(prior); + recordModification(prior, null); // delete everything in created list DirectoryDiff lastDiff = diffs.getLast(); if (lastDiff != null) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeFileUnderConstructionWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeFileUnderConstructionWithSnapshot.java index cbaad7a6309..0b4693a6706 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeFileUnderConstructionWithSnapshot.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeFileUnderConstructionWithSnapshot.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hdfs.server.namenode.INode; import org.apache.hadoop.hdfs.server.namenode.INodeFile; import org.apache.hadoop.hdfs.server.namenode.INodeFileUnderConstruction; +import org.apache.hadoop.hdfs.server.namenode.INodeMap; import org.apache.hadoop.hdfs.server.namenode.Quota; /** @@ -78,7 +79,8 @@ public INodeFile getSnapshotINode(Snapshot snapshot) { @Override public INodeFileUnderConstructionWithSnapshot recordModification( - final Snapshot latest) throws QuotaExceededException { + final Snapshot latest, final INodeMap inodeMap) + throws QuotaExceededException { if (isInLatestSnapshot(latest) && !shouldRecordInSrcSnapshot(latest)) { diffs.saveSelf2Snapshot(latest, this, null); } @@ -100,7 +102,7 @@ public Quota.Counts cleanSubtree(final Snapshot snapshot, Snapshot prior, final BlocksMapUpdateInfo collectedBlocks, final List removedINodes) throws QuotaExceededException { if (snapshot == null) { // delete the current file - recordModification(prior); + recordModification(prior, null); isCurrentFileDeleted = true; Util.collectBlocksAndClear(this, collectedBlocks, removedINodes); return Quota.Counts.newInstance(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeFileWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeFileWithSnapshot.java index 0fde597dc3a..6497584ac06 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeFileWithSnapshot.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/INodeFileWithSnapshot.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.namenode.INode; import org.apache.hadoop.hdfs.server.namenode.INodeFile; +import org.apache.hadoop.hdfs.server.namenode.INodeMap; import org.apache.hadoop.hdfs.server.namenode.Quota; /** @@ -65,8 +66,8 @@ public INodeFile getSnapshotINode(Snapshot snapshot) { } @Override - public INodeFileWithSnapshot recordModification(final Snapshot latest) - throws QuotaExceededException { + public INodeFileWithSnapshot recordModification(final Snapshot latest, + final INodeMap inodeMap) throws QuotaExceededException { if (isInLatestSnapshot(latest) && !shouldRecordInSrcSnapshot(latest)) { diffs.saveSelf2Snapshot(latest, this, null); } @@ -88,7 +89,7 @@ public Quota.Counts cleanSubtree(final Snapshot snapshot, Snapshot prior, final BlocksMapUpdateInfo collectedBlocks, final List removedINodes) throws QuotaExceededException { if (snapshot == null) { // delete the current file - recordModification(prior); + recordModification(prior, null); isCurrentFileDeleted = true; Util.collectBlocksAndClear(this, collectedBlocks, removedINodes); return Quota.Counts.newInstance(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java index 89a682eed4e..613db3b9520 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java @@ -82,7 +82,8 @@ public void setSnapshottable(final String path) throws IOException { s = (INodeDirectorySnapshottable)d; s.setSnapshotQuota(INodeDirectorySnapshottable.SNAPSHOT_LIMIT); } else { - s = d.replaceSelf4INodeDirectorySnapshottable(iip.getLatestSnapshot()); + s = d.replaceSelf4INodeDirectorySnapshottable(iip.getLatestSnapshot(), + fsdir.getINodeMap()); } addSnapshottable(s); } @@ -124,7 +125,7 @@ public void resetSnapshottable(final String path) throws IOException { if (s == fsdir.getRoot()) { s.setSnapshotQuota(0); } else { - s.replaceSelf(iip.getLatestSnapshot()); + s.replaceSelf(iip.getLatestSnapshot(), fsdir.getINodeMap()); } removeSnapshottable(s); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java index 773f3ade3d8..f314da0b3ad 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java @@ -784,7 +784,7 @@ private INode createTreeOfInodes(String path) throws QuotaExceededException { } System.out.println("Adding component " + DFSUtil.bytes2String(component)); dir = new INodeDirectory(++id, component, permstatus, 0); - prev.addChild(dir, false, null); + prev.addChild(dir, false, null, null); prev = dir; } return dir; // Last Inode in the chain diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java index a9b7f25d4af..c61846287d8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java @@ -52,6 +52,7 @@ import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.INode; import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; +import org.apache.hadoop.hdfs.server.namenode.INodeMap; import org.apache.hadoop.hdfs.server.namenode.INodeReference; import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithCount; import org.apache.hadoop.hdfs.server.namenode.snapshot.FileWithSnapshot.FileDiff; @@ -1251,9 +1252,9 @@ public void testRenameUndo() throws Exception { INodeDirectory dir2 = fsdir.getINode4Write(sdir2.toString()).asDirectory(); INodeDirectory mockDir2 = spy(dir2); doReturn(false).when(mockDir2).addChild((INode) anyObject(), anyBoolean(), - (Snapshot) anyObject()); + (Snapshot) anyObject(), (INodeMap) anyObject()); INodeDirectory root = fsdir.getINode4Write("/").asDirectory(); - root.replaceChild(dir2, mockDir2); + root.replaceChild(dir2, mockDir2, fsdir.getINodeMap()); final Path newfoo = new Path(sdir2, "foo"); boolean result = hdfs.rename(foo, newfoo); @@ -1319,9 +1320,9 @@ public void testRenameUndo_2() throws Exception { INodeDirectory dir2 = fsdir.getINode4Write(sdir2.toString()).asDirectory(); INodeDirectory mockDir2 = spy(dir2); doReturn(false).when(mockDir2).addChild((INode) anyObject(), anyBoolean(), - (Snapshot) anyObject()); + (Snapshot) anyObject(), (INodeMap) anyObject()); INodeDirectory root = fsdir.getINode4Write("/").asDirectory(); - root.replaceChild(dir2, mockDir2); + root.replaceChild(dir2, mockDir2, fsdir.getINodeMap()); final Path newfoo = new Path(sdir2, "foo"); boolean result = hdfs.rename(foo, newfoo); @@ -1381,9 +1382,9 @@ public void testRenameUndo_3() throws Exception { INodeDirectory dir3 = fsdir.getINode4Write(sdir3.toString()).asDirectory(); INodeDirectory mockDir3 = spy(dir3); doReturn(false).when(mockDir3).addChild((INode) anyObject(), anyBoolean(), - (Snapshot) anyObject()); + (Snapshot) anyObject(), (INodeMap) anyObject()); INodeDirectory root = fsdir.getINode4Write("/").asDirectory(); - root.replaceChild(dir3, mockDir3); + root.replaceChild(dir3, mockDir3, fsdir.getINodeMap()); final Path foo_dir2 = new Path(sdir2, "foo"); final Path foo_dir3 = new Path(sdir3, "foo"); @@ -1483,11 +1484,12 @@ public void testRenameUndo_4() throws Exception { INodeDirectory mockDir3 = spy(dir3); // fail the rename but succeed in undo doReturn(false).when(mockDir3).addChild((INode) Mockito.isNull(), - anyBoolean(), (Snapshot) anyObject()); - Mockito.when(mockDir3.addChild((INode) Mockito.isNotNull(), anyBoolean(), - (Snapshot) anyObject())).thenReturn(false).thenCallRealMethod(); + anyBoolean(), (Snapshot) anyObject(), (INodeMap) anyObject()); + Mockito.when(mockDir3.addChild((INode) Mockito.isNotNull(), + anyBoolean(), (Snapshot) anyObject(), + (INodeMap) anyObject())).thenReturn(false).thenCallRealMethod(); INodeDirectory root = fsdir.getINode4Write("/").asDirectory(); - root.replaceChild(dir3, mockDir3); + root.replaceChild(dir3, mockDir3, fsdir.getINodeMap()); foo3Node.setParent(mockDir3); try {