Apache/hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-6651. Deletion failure can leak inodes permanently. Contributed by Jing Zhao.

Browse Source
This commit is contained in:
Haohui Mai 2015-02-02 16:32:38 -08:00
parent a3a7efe748
commit 7cb70d3447
21 changed files with 193 additions and 393 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -545,6 +545,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7696. In FsDatasetImpl, the getBlockInputStream(..) and HDFS-7696. In FsDatasetImpl, the getBlockInputStream(..) and
getTmpInputStreams(..) methods may leak file descriptors. (szetszwo) getTmpInputStreams(..) methods may leak file descriptors. (szetszwo)
HDFS-6651. Deletion failure can leak inodes permanently.
(Jing Zhao via wheat9)
Release 2.6.1 - UNRELEASED Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

11
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirDeleteOp.java
View File

@ -19,7 +19,6 @@
import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException; import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo; import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
import org.apache.hadoop.util.ChunkedArrayList; import org.apache.hadoop.util.ChunkedArrayList;
@ -201,7 +200,7 @@ private static boolean deleteAllowed(final INodesInPath iip,
*/ */
private static long unprotectedDelete( private static long unprotectedDelete(
FSDirectory fsd, INodesInPath iip, BlocksMapUpdateInfo collectedBlocks, FSDirectory fsd, INodesInPath iip, BlocksMapUpdateInfo collectedBlocks,
List<INode> removedINodes, long mtime) throws QuotaExceededException { List<INode> removedINodes, long mtime) {
assert fsd.hasWriteLock(); assert fsd.hasWriteLock();
// check if target node exists // check if target node exists
@ -234,12 +233,10 @@ private static long unprotectedDelete(
targetNode.destroyAndCollectBlocks(collectedBlocks, removedINodes); targetNode.destroyAndCollectBlocks(collectedBlocks, removedINodes);
} else { } else {
Quota.Counts counts = targetNode.cleanSubtree(CURRENT_STATE_ID, Quota.Counts counts = targetNode.cleanSubtree(CURRENT_STATE_ID,
latestSnapshot, collectedBlocks, removedINodes, true); latestSnapshot, collectedBlocks, removedINodes);
removed = counts.get(Quota.NAMESPACE); removed = counts.get(Quota.NAMESPACE);
// TODO: quota verification may fail the deletion here. We should not fsd.updateCountNoQuotaCheck(iip, iip.length() - 1,
// count the snapshot diff into quota usage in the future. -counts.get(Quota.NAMESPACE), -counts.get(Quota.DISKSPACE));
fsd.updateCount(iip, -counts.get(Quota.NAMESPACE),
-counts.get(Quota.DISKSPACE), true);
} }
if (NameNode.stateChangeLog.isDebugEnabled()) { if (NameNode.stateChangeLog.isDebugEnabled()) {

8
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
View File

@ -633,7 +633,7 @@ long removeSrc() throws IOException {
} }
} }
boolean removeSrc4OldRename() throws IOException { boolean removeSrc4OldRename() {
final long removedSrc = fsd.removeLastINode(srcIIP); final long removedSrc = fsd.removeLastINode(srcIIP);
if (removedSrc == -1) { if (removedSrc == -1) {
NameNode.stateChangeLog.warn("DIR* FSDirRenameOp.unprotectedRenameTo: " NameNode.stateChangeLog.warn("DIR* FSDirRenameOp.unprotectedRenameTo: "
@ -648,7 +648,7 @@ boolean removeSrc4OldRename() throws IOException {
} }
} }
long removeDst() throws IOException { long removeDst() {
long removedNum = fsd.removeLastINode(dstIIP); long removedNum = fsd.removeLastINode(dstIIP);
if (removedNum != -1) { if (removedNum != -1) {
oldDstChild = dstIIP.getLastINode(); oldDstChild = dstIIP.getLastINode();
@ -735,8 +735,8 @@ boolean cleanDst(BlocksMapUpdateInfo collectedBlocks)
filesDeleted = true; filesDeleted = true;
} else { } else {
filesDeleted = oldDstChild.cleanSubtree(Snapshot.CURRENT_STATE_ID, filesDeleted = oldDstChild.cleanSubtree(Snapshot.CURRENT_STATE_ID,
dstIIP.getLatestSnapshotId(), collectedBlocks, removedINodes, dstIIP.getLatestSnapshotId(), collectedBlocks, removedINodes)
true).get(Quota.NAMESPACE) >= 0; .get(Quota.NAMESPACE) >= 0;
} }
fsd.getFSNamesystem().removeLeasesAndINodes(src, removedINodes, false); fsd.getFSNamesystem().removeLeasesAndINodes(src, removedINodes, false);
return filesDeleted; return filesDeleted;

17
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
View File

@ -607,13 +607,12 @@ void updateSpaceConsumed(INodesInPath iip, long nsDelta, long dsDelta)
* when image/edits have been loaded and the file/dir to be deleted is not * when image/edits have been loaded and the file/dir to be deleted is not
* contained in snapshots. * contained in snapshots.
*/ */
void updateCountForDelete(final INode inode, final INodesInPath iip) void updateCountForDelete(final INode inode, final INodesInPath iip) {
throws QuotaExceededException {
if (getFSNamesystem().isImageLoaded() && if (getFSNamesystem().isImageLoaded() &&
!inode.isInLatestSnapshot(iip.getLatestSnapshotId())) { !inode.isInLatestSnapshot(iip.getLatestSnapshotId())) {
Quota.Counts counts = inode.computeQuotaUsage(); Quota.Counts counts = inode.computeQuotaUsage();
updateCount(iip, -counts.get(Quota.NAMESPACE), unprotectedUpdateCount(iip, iip.length() - 1,
-counts.get(Quota.DISKSPACE), false); -counts.get(Quota.NAMESPACE), -counts.get(Quota.DISKSPACE));
} }
} }
@ -652,8 +651,8 @@ private void updateCount(INodesInPath iip, int numOfINodes,
* update quota of each inode and check to see if quota is exceeded. * update quota of each inode and check to see if quota is exceeded.
* See {@link #updateCount(INodesInPath, long, long, boolean)} * See {@link #updateCount(INodesInPath, long, long, boolean)}
*/ */
private void updateCountNoQuotaCheck(INodesInPath inodesInPath, void updateCountNoQuotaCheck(INodesInPath inodesInPath, int numOfINodes,
int numOfINodes, long nsDelta, long dsDelta) { long nsDelta, long dsDelta) {
assert hasWriteLock(); assert hasWriteLock();
try { try {
updateCount(inodesInPath, numOfINodes, nsDelta, dsDelta, false); updateCount(inodesInPath, numOfINodes, nsDelta, dsDelta, false);
@ -851,7 +850,8 @@ void verifyMaxDirItems(INodeDirectory parent, String parentPath)
* Add a child to the end of the path specified by INodesInPath. * Add a child to the end of the path specified by INodesInPath.
* @return an INodesInPath instance containing the new INode * @return an INodesInPath instance containing the new INode
*/ */
INodesInPath addLastINode(INodesInPath existing, INode inode, @VisibleForTesting
public INodesInPath addLastINode(INodesInPath existing, INode inode,
boolean checkQuota) throws QuotaExceededException { boolean checkQuota) throws QuotaExceededException {
assert existing.getLastINode() != null && assert existing.getLastINode() != null &&
existing.getLastINode().isDirectory(); existing.getLastINode().isDirectory();
@ -926,7 +926,8 @@ INodesInPath addLastINodeNoQuotaCheck(INodesInPath existing, INode i) {
* reference nodes; * reference nodes;
* 1 otherwise. * 1 otherwise.
*/ */
long removeLastINode(final INodesInPath iip) throws QuotaExceededException { @VisibleForTesting
public long removeLastINode(final INodesInPath iip) {
final int latestSnapshot = iip.getLatestSnapshotId(); final int latestSnapshot = iip.getLatestSnapshotId();
final INode last = iip.getLastINode(); final INode last = iip.getLastINode();
final INodeDirectory parent = iip.getINode(-2).asDirectory(); final INodeDirectory parent = iip.getINode(-2).asDirectory();

48
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
View File

@ -20,7 +20,6 @@
import java.io.PrintStream; import java.io.PrintStream;
import java.io.PrintWriter; import java.io.PrintWriter;
import java.io.StringWriter; import java.io.StringWriter;
import java.nio.charset.Charset;
import java.util.List; import java.util.List;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
@ -97,8 +96,7 @@ public final String getUserName() {
abstract void setUser(String user); abstract void setUser(String user);
/** Set user */ /** Set user */
final INode setUser(String user, int latestSnapshotId) final INode setUser(String user, int latestSnapshotId) {
throws QuotaExceededException {
recordModification(latestSnapshotId); recordModification(latestSnapshotId);
setUser(user); setUser(user);
return this; return this;
@ -122,8 +120,7 @@ public final String getGroupName() {
abstract void setGroup(String group); abstract void setGroup(String group);
/** Set group */ /** Set group */
final INode setGroup(String group, int latestSnapshotId) final INode setGroup(String group, int latestSnapshotId) {
throws QuotaExceededException {
recordModification(latestSnapshotId); recordModification(latestSnapshotId);
setGroup(group); setGroup(group);
return this; return this;
@ -148,8 +145,7 @@ public final FsPermission getFsPermission() {
abstract void setPermission(FsPermission permission); abstract void setPermission(FsPermission permission);
/** Set the {@link FsPermission} of this {@link INode} */ /** Set the {@link FsPermission} of this {@link INode} */
INode setPermission(FsPermission permission, int latestSnapshotId) INode setPermission(FsPermission permission, int latestSnapshotId) {
throws QuotaExceededException {
recordModification(latestSnapshotId); recordModification(latestSnapshotId);
setPermission(permission); setPermission(permission);
return this; return this;
@ -164,8 +160,7 @@ public final AclFeature getAclFeature() {
abstract void addAclFeature(AclFeature aclFeature); abstract void addAclFeature(AclFeature aclFeature);
final INode addAclFeature(AclFeature aclFeature, int latestSnapshotId) final INode addAclFeature(AclFeature aclFeature, int latestSnapshotId) {
throws QuotaExceededException {
recordModification(latestSnapshotId); recordModification(latestSnapshotId);
addAclFeature(aclFeature); addAclFeature(aclFeature);
return this; return this;
@ -173,8 +168,7 @@ final INode addAclFeature(AclFeature aclFeature, int latestSnapshotId)
abstract void removeAclFeature(); abstract void removeAclFeature();
final INode removeAclFeature(int latestSnapshotId) final INode removeAclFeature(int latestSnapshotId) {
throws QuotaExceededException {
recordModification(latestSnapshotId); recordModification(latestSnapshotId);
removeAclFeature(); removeAclFeature();
return this; return this;
@ -199,8 +193,7 @@ public final XAttrFeature getXAttrFeature() {
*/ */
abstract void addXAttrFeature(XAttrFeature xAttrFeature); abstract void addXAttrFeature(XAttrFeature xAttrFeature);
final INode addXAttrFeature(XAttrFeature xAttrFeature, int latestSnapshotId) final INode addXAttrFeature(XAttrFeature xAttrFeature, int latestSnapshotId) {
throws QuotaExceededException {
recordModification(latestSnapshotId); recordModification(latestSnapshotId);
addXAttrFeature(xAttrFeature); addXAttrFeature(xAttrFeature);
return this; return this;
@ -211,8 +204,7 @@ final INode addXAttrFeature(XAttrFeature xAttrFeature, int latestSnapshotId)
*/ */
abstract void removeXAttrFeature(); abstract void removeXAttrFeature();
final INode removeXAttrFeature(int lastestSnapshotId) final INode removeXAttrFeature(int lastestSnapshotId) {
throws QuotaExceededException {
recordModification(lastestSnapshotId); recordModification(lastestSnapshotId);
removeXAttrFeature(); removeXAttrFeature();
return this; return this;
@ -228,8 +220,7 @@ public INodeAttributes getSnapshotINode(final int snapshotId) {
/** Is this inode in the latest snapshot? */ /** Is this inode in the latest snapshot? */
public final boolean isInLatestSnapshot(final int latestSnapshotId) { public final boolean isInLatestSnapshot(final int latestSnapshotId) {
if (latestSnapshotId == Snapshot.CURRENT_STATE_ID || if (latestSnapshotId == Snapshot.CURRENT_STATE_ID || latestSnapshotId == Snapshot.NO_SNAPSHOT_ID) {
latestSnapshotId == Snapshot.NO_SNAPSHOT_ID) {
return false; return false;
} }
// if parent is a reference node, parent must be a renamed node. We can // if parent is a reference node, parent must be a renamed node. We can
@ -244,15 +235,12 @@ public final boolean isInLatestSnapshot(final int latestSnapshotId) {
if (!parentDir.isInLatestSnapshot(latestSnapshotId)) { if (!parentDir.isInLatestSnapshot(latestSnapshotId)) {
return false; return false;
} }
final INode child = parentDir.getChild(getLocalNameBytes(), final INode child = parentDir.getChild(getLocalNameBytes(), latestSnapshotId);
latestSnapshotId);
if (this == child) { if (this == child) {
return true; return true;
} }
if (child == null || !(child.isReference())) { return child != null && child.isReference() &&
return false; this == child.asReference().getReferredINode();
}
return this == child.asReference().getReferredINode();
} }
/** @return true if the given inode is an ancestor directory of this inode. */ /** @return true if the given inode is an ancestor directory of this inode. */
@ -302,8 +290,7 @@ public final boolean shouldRecordInSrcSnapshot(final int latestInDst) {
* Note that it is {@link Snapshot#CURRENT_STATE_ID} * Note that it is {@link Snapshot#CURRENT_STATE_ID}
* if no snapshots have been taken. * if no snapshots have been taken.
*/ */
abstract void recordModification(final int latestSnapshotId) abstract void recordModification(final int latestSnapshotId);
throws QuotaExceededException;
/** Check whether it's a reference. */ /** Check whether it's a reference. */
public boolean isReference() { public boolean isReference() {
@ -416,8 +403,7 @@ public INodeSymlink asSymlink() {
*/ */
public abstract Quota.Counts cleanSubtree(final int snapshotId, public abstract Quota.Counts cleanSubtree(final int snapshotId,
int priorSnapshotId, BlocksMapUpdateInfo collectedBlocks, int priorSnapshotId, BlocksMapUpdateInfo collectedBlocks,
List<INode> removedINodes, boolean countDiffChange) List<INode> removedINodes);
throws QuotaExceededException;
/** /**
* Destroy self and clear everything! If the INode is a file, this method * Destroy self and clear everything! If the INode is a file, this method
@ -643,15 +629,14 @@ public final long getModificationTime() {
} }
/** Update modification time if it is larger than the current value. */ /** Update modification time if it is larger than the current value. */
public abstract INode updateModificationTime(long mtime, int latestSnapshotId) public abstract INode updateModificationTime(long mtime, int latestSnapshotId);
throws QuotaExceededException;
/** Set the last modification time of inode. */ /** Set the last modification time of inode. */
public abstract void setModificationTime(long modificationTime); public abstract void setModificationTime(long modificationTime);
/** Set the last modification time of inode. */ /** Set the last modification time of inode. */
public final INode setModificationTime(long modificationTime, public final INode setModificationTime(long modificationTime,
int latestSnapshotId) throws QuotaExceededException { int latestSnapshotId) {
recordModification(latestSnapshotId); recordModification(latestSnapshotId);
setModificationTime(modificationTime); setModificationTime(modificationTime);
return this; return this;
@ -680,8 +665,7 @@ public final long getAccessTime() {
/** /**
* Set last access time of inode. * Set last access time of inode.
*/ */
public final INode setAccessTime(long accessTime, int latestSnapshotId) public final INode setAccessTime(long accessTime, int latestSnapshotId) {
throws QuotaExceededException {
recordModification(latestSnapshotId); recordModification(latestSnapshotId);
setAccessTime(accessTime); setAccessTime(accessTime);
return this; return this;

21
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
View File

@ -28,7 +28,6 @@
import org.apache.hadoop.fs.PathIsNotDirectoryException; import org.apache.hadoop.fs.PathIsNotDirectoryException;
import org.apache.hadoop.fs.XAttr; import org.apache.hadoop.fs.XAttr;
import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException; import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.protocol.SnapshotException; import org.apache.hadoop.hdfs.protocol.SnapshotException;
@ -352,8 +351,7 @@ INodeReference.WithName replaceChild4ReferenceWithName(INode oldChild,
} }
@Override @Override
public void recordModification(int latestSnapshotId) public void recordModification(int latestSnapshotId) {
throws QuotaExceededException {
if (isInLatestSnapshot(latestSnapshotId) if (isInLatestSnapshot(latestSnapshotId)
&& !shouldRecordInSrcSnapshot(latestSnapshotId)) { && !shouldRecordInSrcSnapshot(latestSnapshotId)) {
// add snapshot feature if necessary // add snapshot feature if necessary
@ -372,7 +370,7 @@ public void recordModification(int latestSnapshotId)
* @return the child inode, which may be replaced. * @return the child inode, which may be replaced.
*/ */
public INode saveChild2Snapshot(final INode child, final int latestSnapshotId, public INode saveChild2Snapshot(final INode child, final int latestSnapshotId,
final INode snapshotCopy) throws QuotaExceededException { final INode snapshotCopy) {
if (latestSnapshotId == Snapshot.CURRENT_STATE_ID) { if (latestSnapshotId == Snapshot.CURRENT_STATE_ID) {
return child; return child;
} }
@ -471,8 +469,7 @@ static int nextChild(ReadOnlyList<INode> children, byte[] name) {
/** /**
* Remove the specified child from this directory. * Remove the specified child from this directory.
*/ */
public boolean removeChild(INode child, int latestSnapshotId) public boolean removeChild(INode child, int latestSnapshotId) {
throws QuotaExceededException {
if (isInLatestSnapshot(latestSnapshotId)) { if (isInLatestSnapshot(latestSnapshotId)) {
// create snapshot feature if necessary // create snapshot feature if necessary
DirectoryWithSnapshotFeature sf = this.getDirectoryWithSnapshotFeature(); DirectoryWithSnapshotFeature sf = this.getDirectoryWithSnapshotFeature();
@ -737,8 +734,7 @@ public void clear() {
/** Call cleanSubtree(..) recursively down the subtree. */ /** Call cleanSubtree(..) recursively down the subtree. */
public Quota.Counts cleanSubtreeRecursively(final int snapshot, public Quota.Counts cleanSubtreeRecursively(final int snapshot,
int prior, final BlocksMapUpdateInfo collectedBlocks, int prior, final BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes, final Map<INode, INode> excludedNodes, final List<INode> removedINodes, final Map<INode, INode> excludedNodes) {
final boolean countDiffChange) throws QuotaExceededException {
Quota.Counts counts = Quota.Counts.newInstance(); Quota.Counts counts = Quota.Counts.newInstance();
// in case of deletion snapshot, since this call happens after we modify // in case of deletion snapshot, since this call happens after we modify
// the diff list, the snapshot to be deleted has been combined or renamed // the diff list, the snapshot to be deleted has been combined or renamed
@ -753,7 +749,7 @@ public Quota.Counts cleanSubtreeRecursively(final int snapshot,
continue; continue;
} else { } else {
Quota.Counts childCounts = child.cleanSubtree(snapshot, prior, Quota.Counts childCounts = child.cleanSubtree(snapshot, prior,
collectedBlocks, removedINodes, countDiffChange); collectedBlocks, removedINodes);
counts.add(childCounts); counts.add(childCounts);
} }
} }
@ -780,13 +776,12 @@ public void destroyAndCollectBlocks(final BlocksMapUpdateInfo collectedBlocks,
@Override @Override
public Quota.Counts cleanSubtree(final int snapshotId, int priorSnapshotId, public Quota.Counts cleanSubtree(final int snapshotId, int priorSnapshotId,
final BlocksMapUpdateInfo collectedBlocks, final BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes, final boolean countDiffChange) final List<INode> removedINodes) {
throws QuotaExceededException {
DirectoryWithSnapshotFeature sf = getDirectoryWithSnapshotFeature(); DirectoryWithSnapshotFeature sf = getDirectoryWithSnapshotFeature();
// there is snapshot data // there is snapshot data
if (sf != null) { if (sf != null) {
return sf.cleanDirectory(this, snapshotId, priorSnapshotId, return sf.cleanDirectory(this, snapshotId, priorSnapshotId,
collectedBlocks, removedINodes, countDiffChange); collectedBlocks, removedINodes);
} }
// there is no snapshot data // there is no snapshot data
if (priorSnapshotId == Snapshot.NO_SNAPSHOT_ID if (priorSnapshotId == Snapshot.NO_SNAPSHOT_ID
@ -799,7 +794,7 @@ public Quota.Counts cleanSubtree(final int snapshotId, int priorSnapshotId,
} else { } else {
// process recursively down the subtree // process recursively down the subtree
Quota.Counts counts = cleanSubtreeRecursively(snapshotId, priorSnapshotId, Quota.Counts counts = cleanSubtreeRecursively(snapshotId, priorSnapshotId,
collectedBlocks, removedINodes, null, countDiffChange); collectedBlocks, removedINodes, null);
if (isQuotaSet()) { if (isQuotaSet()) {
getDirectoryWithQuotaFeature().addSpaceConsumed2Cache( getDirectoryWithQuotaFeature().addSpaceConsumed2Cache(
-counts.get(Quota.NAMESPACE), -counts.get(Quota.DISKSPACE)); -counts.get(Quota.NAMESPACE), -counts.get(Quota.DISKSPACE));

13
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
View File

@ -304,13 +304,11 @@ public INodeFileAttributes getSnapshotINode(final int snapshotId) {
} }
@Override @Override
public void recordModification(final int latestSnapshotId) public void recordModification(final int latestSnapshotId) {
throws QuotaExceededException {
recordModification(latestSnapshotId, false); recordModification(latestSnapshotId, false);
} }
public void recordModification(final int latestSnapshotId, boolean withBlocks) public void recordModification(final int latestSnapshotId, boolean withBlocks) {
throws QuotaExceededException {
if (isInLatestSnapshot(latestSnapshotId) if (isInLatestSnapshot(latestSnapshotId)
&& !shouldRecordInSrcSnapshot(latestSnapshotId)) { && !shouldRecordInSrcSnapshot(latestSnapshotId)) {
// the file is in snapshot, create a snapshot feature if it does not have // the file is in snapshot, create a snapshot feature if it does not have
@ -489,12 +487,11 @@ public void setBlocks(BlockInfo[] blocks) {
@Override @Override
public Quota.Counts cleanSubtree(final int snapshot, int priorSnapshotId, public Quota.Counts cleanSubtree(final int snapshot, int priorSnapshotId,
final BlocksMapUpdateInfo collectedBlocks, final BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes, final boolean countDiffChange) final List<INode> removedINodes) {
throws QuotaExceededException {
FileWithSnapshotFeature sf = getFileWithSnapshotFeature(); FileWithSnapshotFeature sf = getFileWithSnapshotFeature();
if (sf != null) { if (sf != null) {
return sf.cleanFile(this, snapshot, priorSnapshotId, collectedBlocks, return sf.cleanFile(this, snapshot, priorSnapshotId, collectedBlocks,
removedINodes, countDiffChange); removedINodes);
} }
Quota.Counts counts = Quota.Counts.newInstance(); Quota.Counts counts = Quota.Counts.newInstance();
if (snapshot == CURRENT_STATE_ID) { if (snapshot == CURRENT_STATE_ID) {
@ -552,11 +549,9 @@ public final Quota.Counts computeQuotaUsage(Quota.Counts counts,
if (sf != null) { if (sf != null) {
FileDiffList fileDiffList = sf.getDiffs(); FileDiffList fileDiffList = sf.getDiffs();
int last = fileDiffList.getLastSnapshotId(); int last = fileDiffList.getLastSnapshotId();
List<FileDiff> diffs = fileDiffList.asList();
if (lastSnapshotId == Snapshot.CURRENT_STATE_ID if (lastSnapshotId == Snapshot.CURRENT_STATE_ID
|| last == Snapshot.CURRENT_STATE_ID) { || last == Snapshot.CURRENT_STATE_ID) {
nsDelta += diffs.size();
dsDelta = diskspaceConsumed(); dsDelta = diskspaceConsumed();
} else if (last < lastSnapshotId) { } else if (last < lastSnapshotId) {
dsDelta = computeFileSize(true, false) * getFileReplication(); dsDelta = computeFileSize(true, false) * getFileReplication();

8
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java
View File

@ -22,8 +22,6 @@
import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite; import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.server.namenode.Quota.Counts; import org.apache.hadoop.hdfs.server.namenode.Quota.Counts;
import org.apache.hadoop.util.GSet; import org.apache.hadoop.util.GSet;
@ -95,8 +93,7 @@ public INode get(long id) {
"", "", new FsPermission((short) 0)), 0, 0) { "", "", new FsPermission((short) 0)), 0, 0) {
@Override @Override
void recordModification(int latestSnapshotId) void recordModification(int latestSnapshotId) {
throws QuotaExceededException {
} }
@Override @Override
@ -119,8 +116,7 @@ public ContentSummaryComputationContext computeContentSummary(
@Override @Override
public Counts cleanSubtree(int snapshotId, int priorSnapshotId, public Counts cleanSubtree(int snapshotId, int priorSnapshotId,
BlocksMapUpdateInfo collectedBlocks, List<INode> removedINodes, BlocksMapUpdateInfo collectedBlocks, List<INode> removedINodes) {
boolean countDiffChange) throws QuotaExceededException {
return null; return null;
} }

34
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java
View File

@ -30,6 +30,7 @@
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
import org.mortbay.log.Log;
/** /**
* An anonymous reference to an inode. * An anonymous reference to an inode.
@ -265,8 +266,7 @@ public final long getModificationTime(int snapshotId) {
} }
@Override @Override
public final INode updateModificationTime(long mtime, int latestSnapshotId) public final INode updateModificationTime(long mtime, int latestSnapshotId) {
throws QuotaExceededException {
return referred.updateModificationTime(mtime, latestSnapshotId); return referred.updateModificationTime(mtime, latestSnapshotId);
} }
@ -296,17 +296,15 @@ public final byte getLocalStoragePolicyID() {
} }
@Override @Override
final void recordModification(int latestSnapshotId) final void recordModification(int latestSnapshotId) {
throws QuotaExceededException {
referred.recordModification(latestSnapshotId); referred.recordModification(latestSnapshotId);
} }
@Override // used by WithCount @Override // used by WithCount
public Quota.Counts cleanSubtree(int snapshot, int prior, public Quota.Counts cleanSubtree(int snapshot, int prior,
BlocksMapUpdateInfo collectedBlocks, final List<INode> removedINodes, BlocksMapUpdateInfo collectedBlocks, final List<INode> removedINodes) {
final boolean countDiffChange) throws QuotaExceededException {
return referred.cleanSubtree(snapshot, prior, collectedBlocks, return referred.cleanSubtree(snapshot, prior, collectedBlocks,
removedINodes, countDiffChange); removedINodes);
} }
@Override // used by WithCount @Override // used by WithCount
@ -537,8 +535,7 @@ public final Quota.Counts computeQuotaUsage(Quota.Counts counts,
@Override @Override
public Quota.Counts cleanSubtree(final int snapshot, int prior, public Quota.Counts cleanSubtree(final int snapshot, int prior,
final BlocksMapUpdateInfo collectedBlocks, final BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes, final boolean countDiffChange) final List<INode> removedINodes) {
throws QuotaExceededException {
// since WithName node resides in deleted list acting as a snapshot copy, // since WithName node resides in deleted list acting as a snapshot copy,
// the parameter snapshot must be non-null // the parameter snapshot must be non-null
Preconditions.checkArgument(snapshot != Snapshot.CURRENT_STATE_ID); Preconditions.checkArgument(snapshot != Snapshot.CURRENT_STATE_ID);
@ -554,11 +551,15 @@ public Quota.Counts cleanSubtree(final int snapshot, int prior,
} }
Quota.Counts counts = getReferredINode().cleanSubtree(snapshot, prior, Quota.Counts counts = getReferredINode().cleanSubtree(snapshot, prior,
collectedBlocks, removedINodes, false); collectedBlocks, removedINodes);
INodeReference ref = getReferredINode().getParentReference(); INodeReference ref = getReferredINode().getParentReference();
if (ref != null) { if (ref != null) {
try {
ref.addSpaceConsumed(-counts.get(Quota.NAMESPACE), ref.addSpaceConsumed(-counts.get(Quota.NAMESPACE),
-counts.get(Quota.DISKSPACE), true); -counts.get(Quota.DISKSPACE), true);
} catch (QuotaExceededException e) {
Log.warn("Should not have QuotaExceededException");
}
} }
if (snapshot < lastSnapshotId) { if (snapshot < lastSnapshotId) {
@ -597,7 +598,7 @@ public void destroyAndCollectBlocks(BlocksMapUpdateInfo collectedBlocks,
} }
try { try {
Quota.Counts counts = referred.cleanSubtree(snapshot, prior, Quota.Counts counts = referred.cleanSubtree(snapshot, prior,
collectedBlocks, removedINodes, false); collectedBlocks, removedINodes);
INodeReference ref = getReferredINode().getParentReference(); INodeReference ref = getReferredINode().getParentReference();
if (ref != null) { if (ref != null) {
ref.addSpaceConsumed(-counts.get(Quota.NAMESPACE), ref.addSpaceConsumed(-counts.get(Quota.NAMESPACE),
@ -653,8 +654,7 @@ public DstReference(INodeDirectory parent, WithCount referred,
@Override @Override
public Quota.Counts cleanSubtree(int snapshot, int prior, public Quota.Counts cleanSubtree(int snapshot, int prior,
BlocksMapUpdateInfo collectedBlocks, List<INode> removedINodes, BlocksMapUpdateInfo collectedBlocks, List<INode> removedINodes) {
final boolean countDiffChange) throws QuotaExceededException {
if (snapshot == Snapshot.CURRENT_STATE_ID if (snapshot == Snapshot.CURRENT_STATE_ID
&& prior == Snapshot.NO_SNAPSHOT_ID) { && prior == Snapshot.NO_SNAPSHOT_ID) {
Quota.Counts counts = Quota.Counts.newInstance(); Quota.Counts counts = Quota.Counts.newInstance();
@ -676,7 +676,7 @@ public Quota.Counts cleanSubtree(int snapshot, int prior,
return Quota.Counts.newInstance(); return Quota.Counts.newInstance();
} }
return getReferredINode().cleanSubtree(snapshot, prior, return getReferredINode().cleanSubtree(snapshot, prior,
collectedBlocks, removedINodes, countDiffChange); collectedBlocks, removedINodes);
} }
} }
@ -714,15 +714,11 @@ public void destroyAndCollectBlocks(
Preconditions.checkState(file.isWithSnapshot()); Preconditions.checkState(file.isWithSnapshot());
// make sure we mark the file as deleted // make sure we mark the file as deleted
file.getFileWithSnapshotFeature().deleteCurrentFile(); file.getFileWithSnapshotFeature().deleteCurrentFile();
try {
// when calling cleanSubtree of the referred node, since we // when calling cleanSubtree of the referred node, since we
// compute quota usage updates before calling this destroy // compute quota usage updates before calling this destroy
// function, we use true for countDiffChange // function, we use true for countDiffChange
referred.cleanSubtree(snapshot, prior, collectedBlocks, referred.cleanSubtree(snapshot, prior, collectedBlocks,
removedINodes, true); removedINodes);
} catch (QuotaExceededException e) {
LOG.error("should not exceed quota while snapshot deletion", e);
}
} else if (referred.isDirectory()) { } else if (referred.isDirectory()) {
// similarly, if referred is a directory, it must be an // similarly, if referred is a directory, it must be an
// INodeDirectory with snapshot // INodeDirectory with snapshot

7
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java
View File

@ -23,10 +23,7 @@
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.hdfs.server.namenode.AclFeature;
import org.apache.hadoop.hdfs.server.namenode.XAttrFeature;
/** /**
* An {@link INode} representing a symbolic link. * An {@link INode} representing a symbolic link.
@ -47,7 +44,7 @@ public class INodeSymlink extends INodeWithAdditionalFields {
} }
@Override @Override
void recordModification(int latestSnapshotId) throws QuotaExceededException { void recordModification(int latestSnapshotId) {
if (isInLatestSnapshot(latestSnapshotId)) { if (isInLatestSnapshot(latestSnapshotId)) {
INodeDirectory parent = getParent(); INodeDirectory parent = getParent();
parent.saveChild2Snapshot(this, latestSnapshotId, new INodeSymlink(this)); parent.saveChild2Snapshot(this, latestSnapshotId, new INodeSymlink(this));
@ -77,7 +74,7 @@ public byte[] getSymlink() {
@Override @Override
public Quota.Counts cleanSubtree(final int snapshotId, int priorSnapshotId, public Quota.Counts cleanSubtree(final int snapshotId, int priorSnapshotId,
final BlocksMapUpdateInfo collectedBlocks, final BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes, final boolean countDiffChange) { final List<INode> removedINodes) {
if (snapshotId == Snapshot.CURRENT_STATE_ID if (snapshotId == Snapshot.CURRENT_STATE_ID
&& priorSnapshotId == Snapshot.NO_SNAPSHOT_ID) { && priorSnapshotId == Snapshot.NO_SNAPSHOT_ID) {
destroyAndCollectBlocks(collectedBlocks, removedINodes); destroyAndCollectBlocks(collectedBlocks, removedINodes);

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java
View File

@ -20,7 +20,6 @@
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.hdfs.util.LongBitFormat; import org.apache.hadoop.hdfs.util.LongBitFormat;
import org.apache.hadoop.util.LightWeightGSet.LinkedElement; import org.apache.hadoop.util.LightWeightGSet.LinkedElement;
@ -239,8 +238,7 @@ final long getModificationTime(int snapshotId) {
/** Update modification time if it is larger than the current value. */ /** Update modification time if it is larger than the current value. */
@Override @Override
public final INode updateModificationTime(long mtime, int latestSnapshotId) public final INode updateModificationTime(long mtime, int latestSnapshotId) {
throws QuotaExceededException {
Preconditions.checkState(isDirectory()); Preconditions.checkState(isDirectory());
if (mtime <= modificationTime) { if (mtime <= modificationTime) {
return this; return this;

42
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java
View File

@ -22,8 +22,6 @@
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.server.namenode.INode; import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo; import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
import org.apache.hadoop.hdfs.server.namenode.INodeAttributes; import org.apache.hadoop.hdfs.server.namenode.INodeAttributes;
@ -71,8 +69,7 @@ public void clear() {
public final Quota.Counts deleteSnapshotDiff(final int snapshot, public final Quota.Counts deleteSnapshotDiff(final int snapshot,
final int prior, final N currentINode, final int prior, final N currentINode,
final BlocksMapUpdateInfo collectedBlocks, final BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes, boolean countDiffChange) final List<INode> removedINodes) {
throws QuotaExceededException {
int snapshotIndex = Collections.binarySearch(diffs, snapshot); int snapshotIndex = Collections.binarySearch(diffs, snapshot);
Quota.Counts counts = Quota.Counts.newInstance(); Quota.Counts counts = Quota.Counts.newInstance();
@ -83,14 +80,6 @@ public final Quota.Counts deleteSnapshotDiff(final int snapshot,
diffs.get(snapshotIndex).setSnapshotId(prior); diffs.get(snapshotIndex).setSnapshotId(prior);
} else { // there is no snapshot before } else { // there is no snapshot before
removed = diffs.remove(0); removed = diffs.remove(0);
if (countDiffChange) {
counts.add(Quota.NAMESPACE, 1);
} else {
// the currentINode must be a descendant of a WithName node, which set
// countDiffChange to false. In that case we should count in the diff
// change when updating the quota usage in the current tree
currentINode.addSpaceConsumed(-1, 0, false);
}
counts.add(removed.destroyDiffAndCollectBlocks(currentINode, counts.add(removed.destroyDiffAndCollectBlocks(currentINode,
collectedBlocks, removedINodes)); collectedBlocks, removedINodes));
} }
@ -101,11 +90,6 @@ public final Quota.Counts deleteSnapshotDiff(final int snapshot,
} else { } else {
// combine the to-be-removed diff with its previous diff // combine the to-be-removed diff with its previous diff
removed = diffs.remove(snapshotIndex); removed = diffs.remove(snapshotIndex);
if (countDiffChange) {
counts.add(Quota.NAMESPACE, 1);
} else {
currentINode.addSpaceConsumed(-1, 0, false);
}
if (previous.snapshotINode == null) { if (previous.snapshotINode == null) {
previous.snapshotINode = removed.snapshotINode; previous.snapshotINode = removed.snapshotINode;
} }
@ -120,9 +104,7 @@ public final Quota.Counts deleteSnapshotDiff(final int snapshot,
} }
/** Add an {@link AbstractINodeDiff} for the given snapshot. */ /** Add an {@link AbstractINodeDiff} for the given snapshot. */
final D addDiff(int latestSnapshotId, N currentINode) final D addDiff(int latestSnapshotId, N currentINode) {
throws QuotaExceededException {
currentINode.addSpaceConsumed(1, 0, true);
return addLast(createDiff(latestSnapshotId, currentINode)); return addLast(createDiff(latestSnapshotId, currentINode));
} }
@ -275,26 +257,16 @@ public A getSnapshotINode(final int snapshotId, final A currentINode) {
* Check if the latest snapshot diff exists. If not, add it. * Check if the latest snapshot diff exists. If not, add it.
* @return the latest snapshot diff, which is never null. * @return the latest snapshot diff, which is never null.
*/ */
final D checkAndAddLatestSnapshotDiff(int latestSnapshotId, N currentINode) final D checkAndAddLatestSnapshotDiff(int latestSnapshotId, N currentINode) {
throws QuotaExceededException {
final D last = getLast(); final D last = getLast();
if (last != null return (last != null && Snapshot.ID_INTEGER_COMPARATOR
&& Snapshot.ID_INTEGER_COMPARATOR.compare(last.getSnapshotId(), .compare(last.getSnapshotId(), latestSnapshotId) >= 0) ?
latestSnapshotId) >= 0) { last : addDiff(latestSnapshotId, currentINode);
return last;
} else {
try {
return addDiff(latestSnapshotId, currentINode);
} catch(NSQuotaExceededException e) {
e.setMessagePrefix("Failed to record modification for snapshot");
throw e;
}
}
} }
/** Save the snapshot copy to the latest snapshot. */ /** Save the snapshot copy to the latest snapshot. */
public D saveSelf2Snapshot(int latestSnapshotId, N currentINode, public D saveSelf2Snapshot(int latestSnapshotId, N currentINode,
A snapshotCopy) throws QuotaExceededException { A snapshotCopy) {
D diff = null; D diff = null;
if (latestSnapshotId != Snapshot.CURRENT_STATE_ID) { if (latestSnapshotId != Snapshot.CURRENT_STATE_ID) {
diff = checkAndAddLatestSnapshotDiff(latestSnapshotId, currentINode); diff = checkAndAddLatestSnapshotDiff(latestSnapshotId, currentINode);

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java
View File

@ -215,7 +215,7 @@ public Snapshot removeSnapshot(INodeDirectory snapshotRoot,
int prior = Snapshot.findLatestSnapshot(snapshotRoot, snapshot.getId()); int prior = Snapshot.findLatestSnapshot(snapshotRoot, snapshot.getId());
try { try {
Quota.Counts counts = snapshotRoot.cleanSubtree(snapshot.getId(), Quota.Counts counts = snapshotRoot.cleanSubtree(snapshot.getId(),
prior, collectedBlocks, removedINodes, true); prior, collectedBlocks, removedINodes);
INodeDirectory parent = snapshotRoot.getParent(); INodeDirectory parent = snapshotRoot.getParent();
if (parent != null) { if (parent != null) {
// there will not be any WithName node corresponding to the deleted // there will not be any WithName node corresponding to the deleted

38
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
View File

@ -70,7 +70,7 @@ private ChildrenDiff(final List<INode> created, final List<INode> deleted) {
* Replace the given child from the created/deleted list. * Replace the given child from the created/deleted list.
* @return true if the child is replaced; false if the child is not found. * @return true if the child is replaced; false if the child is not found.
*/ */
private final boolean replace(final ListType type, private boolean replace(final ListType type,
final INode oldChild, final INode newChild) { final INode oldChild, final INode newChild) {
final List<INode> list = getList(type); final List<INode> list = getList(type);
final int i = search(list, oldChild.getLocalNameBytes()); final int i = search(list, oldChild.getLocalNameBytes());
@ -83,7 +83,7 @@ private final boolean replace(final ListType type,
return true; return true;
} }
private final boolean removeChild(ListType type, final INode child) { private boolean removeChild(ListType type, final INode child) {
final List<INode> list = getList(type); final List<INode> list = getList(type);
final int i = searchIndex(type, child.getLocalNameBytes()); final int i = searchIndex(type, child.getLocalNameBytes());
if (i >= 0 && list.get(i) == child) { if (i >= 0 && list.get(i) == child) {
@ -410,15 +410,14 @@ public static void destroyDstSubtree(INode inode, final int snapshot,
&& snapshot != Snapshot.CURRENT_STATE_ID) { && snapshot != Snapshot.CURRENT_STATE_ID) {
// this inode has been renamed before the deletion of the DstReference // this inode has been renamed before the deletion of the DstReference
// subtree // subtree
inode.cleanSubtree(snapshot, prior, collectedBlocks, removedINodes, inode.cleanSubtree(snapshot, prior, collectedBlocks, removedINodes);
true);
} else { } else {
// for DstReference node, continue this process to its subtree // for DstReference node, continue this process to its subtree
destroyDstSubtree(inode.asReference().getReferredINode(), snapshot, destroyDstSubtree(inode.asReference().getReferredINode(), snapshot,
prior, collectedBlocks, removedINodes); prior, collectedBlocks, removedINodes);
} }
} else if (inode.isFile()) { } else if (inode.isFile()) {
inode.cleanSubtree(snapshot, prior, collectedBlocks, removedINodes, true); inode.cleanSubtree(snapshot, prior, collectedBlocks, removedINodes);
} else if (inode.isDirectory()) { } else if (inode.isDirectory()) {
Map<INode, INode> excludedNodes = null; Map<INode, INode> excludedNodes = null;
INodeDirectory dir = inode.asDirectory(); INodeDirectory dir = inode.asDirectory();
@ -433,7 +432,7 @@ public static void destroyDstSubtree(INode inode, final int snapshot,
if (snapshot != Snapshot.CURRENT_STATE_ID) { if (snapshot != Snapshot.CURRENT_STATE_ID) {
diffList.deleteSnapshotDiff(snapshot, prior, dir, collectedBlocks, diffList.deleteSnapshotDiff(snapshot, prior, dir, collectedBlocks,
removedINodes, true); removedINodes);
} }
priorDiff = diffList.getDiffById(prior); priorDiff = diffList.getDiffById(prior);
if (priorDiff != null && priorDiff.getSnapshotId() == prior) { if (priorDiff != null && priorDiff.getSnapshotId() == prior) {
@ -463,8 +462,7 @@ public static void destroyDstSubtree(INode inode, final int snapshot,
private static Quota.Counts cleanDeletedINode(INode inode, private static Quota.Counts cleanDeletedINode(INode inode,
final int post, final int prior, final int post, final int prior,
final BlocksMapUpdateInfo collectedBlocks, final BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes, final boolean countDiffChange) final List<INode> removedINodes) {
throws QuotaExceededException {
Quota.Counts counts = Quota.Counts.newInstance(); Quota.Counts counts = Quota.Counts.newInstance();
Deque<INode> queue = new ArrayDeque<INode>(); Deque<INode> queue = new ArrayDeque<INode>();
queue.addLast(inode); queue.addLast(inode);
@ -473,15 +471,14 @@ private static Quota.Counts cleanDeletedINode(INode inode,
if (topNode instanceof INodeReference.WithName) { if (topNode instanceof INodeReference.WithName) {
INodeReference.WithName wn = (INodeReference.WithName) topNode; INodeReference.WithName wn = (INodeReference.WithName) topNode;
if (wn.getLastSnapshotId() >= post) { if (wn.getLastSnapshotId() >= post) {
wn.cleanSubtree(post, prior, collectedBlocks, removedINodes, wn.cleanSubtree(post, prior, collectedBlocks, removedINodes);
countDiffChange);
} }
// For DstReference node, since the node is not in the created list of // For DstReference node, since the node is not in the created list of
// prior, we should treat it as regular file/dir // prior, we should treat it as regular file/dir
} else if (topNode.isFile() && topNode.asFile().isWithSnapshot()) { } else if (topNode.isFile() && topNode.asFile().isWithSnapshot()) {
INodeFile file = topNode.asFile(); INodeFile file = topNode.asFile();
counts.add(file.getDiffs().deleteSnapshotDiff(post, prior, file, counts.add(file.getDiffs().deleteSnapshotDiff(post, prior, file,
collectedBlocks, removedINodes, countDiffChange)); collectedBlocks, removedINodes));
} else if (topNode.isDirectory()) { } else if (topNode.isDirectory()) {
INodeDirectory dir = topNode.asDirectory(); INodeDirectory dir = topNode.asDirectory();
ChildrenDiff priorChildrenDiff = null; ChildrenDiff priorChildrenDiff = null;
@ -561,7 +558,7 @@ public boolean addChild(INodeDirectory parent, INode inode,
* needs to make sure that parent is in the given snapshot "latest". * needs to make sure that parent is in the given snapshot "latest".
*/ */
public boolean removeChild(INodeDirectory parent, INode child, public boolean removeChild(INodeDirectory parent, INode child,
int latestSnapshotId) throws QuotaExceededException { int latestSnapshotId) {
// For a directory that is not a renamed node, if isInLatestSnapshot returns // For a directory that is not a renamed node, if isInLatestSnapshot returns
// false, the directory is not in the latest snapshot, thus we do not need // false, the directory is not in the latest snapshot, thus we do not need
// to record the removed child in any snapshot. // to record the removed child in any snapshot.
@ -607,8 +604,7 @@ public INode getChild(INodeDirectory currentINode, byte[] name,
/** Used to record the modification of a symlink node */ /** Used to record the modification of a symlink node */
public INode saveChild2Snapshot(INodeDirectory currentINode, public INode saveChild2Snapshot(INodeDirectory currentINode,
final INode child, final int latestSnapshotId, final INode snapshotCopy) final INode child, final int latestSnapshotId, final INode snapshotCopy) {
throws QuotaExceededException {
Preconditions.checkArgument(!child.isDirectory(), Preconditions.checkArgument(!child.isDirectory(),
"child is a directory, child=%s", child); "child is a directory, child=%s", child);
Preconditions.checkArgument(latestSnapshotId != Snapshot.CURRENT_STATE_ID); Preconditions.checkArgument(latestSnapshotId != Snapshot.CURRENT_STATE_ID);
@ -640,7 +636,6 @@ public Quota.Counts computeQuotaUsage4CurrentDirectory(Quota.Counts counts) {
deleted.computeQuotaUsage(counts, false, Snapshot.CURRENT_STATE_ID); deleted.computeQuotaUsage(counts, false, Snapshot.CURRENT_STATE_ID);
} }
} }
counts.add(Quota.NAMESPACE, diffs.asList().size());
return counts; return counts;
} }
@ -714,8 +709,7 @@ boolean computeDiffBetweenSnapshots(Snapshot fromSnapshot,
public Quota.Counts cleanDirectory(final INodeDirectory currentINode, public Quota.Counts cleanDirectory(final INodeDirectory currentINode,
final int snapshot, int prior, final int snapshot, int prior,
final BlocksMapUpdateInfo collectedBlocks, final BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes, final boolean countDiffChange) final List<INode> removedINodes) {
throws QuotaExceededException {
Quota.Counts counts = Quota.Counts.newInstance(); Quota.Counts counts = Quota.Counts.newInstance();
Map<INode, INode> priorCreated = null; Map<INode, INode> priorCreated = null;
Map<INode, INode> priorDeleted = null; Map<INode, INode> priorDeleted = null;
@ -728,7 +722,7 @@ public Quota.Counts cleanDirectory(final INodeDirectory currentINode,
collectedBlocks, removedINodes)); collectedBlocks, removedINodes));
} }
counts.add(currentINode.cleanSubtreeRecursively(snapshot, prior, counts.add(currentINode.cleanSubtreeRecursively(snapshot, prior,
collectedBlocks, removedINodes, priorDeleted, countDiffChange)); collectedBlocks, removedINodes, priorDeleted));
} else { } else {
// update prior // update prior
prior = getDiffs().updatePrior(snapshot, prior); prior = getDiffs().updatePrior(snapshot, prior);
@ -745,9 +739,9 @@ public Quota.Counts cleanDirectory(final INodeDirectory currentINode,
} }
counts.add(getDiffs().deleteSnapshotDiff(snapshot, prior, counts.add(getDiffs().deleteSnapshotDiff(snapshot, prior,
currentINode, collectedBlocks, removedINodes, countDiffChange)); currentINode, collectedBlocks, removedINodes));
counts.add(currentINode.cleanSubtreeRecursively(snapshot, prior, counts.add(currentINode.cleanSubtreeRecursively(snapshot, prior,
collectedBlocks, removedINodes, priorDeleted, countDiffChange)); collectedBlocks, removedINodes, priorDeleted));
// check priorDiff again since it may be created during the diff deletion // check priorDiff again since it may be created during the diff deletion
if (prior != Snapshot.NO_SNAPSHOT_ID) { if (prior != Snapshot.NO_SNAPSHOT_ID) {
@ -764,7 +758,7 @@ public Quota.Counts cleanDirectory(final INodeDirectory currentINode,
ListType.CREATED)) { ListType.CREATED)) {
if (priorCreated.containsKey(cNode)) { if (priorCreated.containsKey(cNode)) {
counts.add(cNode.cleanSubtree(snapshot, Snapshot.NO_SNAPSHOT_ID, counts.add(cNode.cleanSubtree(snapshot, Snapshot.NO_SNAPSHOT_ID,
collectedBlocks, removedINodes, countDiffChange)); collectedBlocks, removedINodes));
} }
} }
} }
@ -781,7 +775,7 @@ public Quota.Counts cleanDirectory(final INodeDirectory currentINode,
ListType.DELETED)) { ListType.DELETED)) {
if (priorDeleted == null || !priorDeleted.containsKey(dNode)) { if (priorDeleted == null || !priorDeleted.containsKey(dNode)) {
counts.add(cleanDeletedINode(dNode, snapshot, prior, counts.add(cleanDeletedINode(dNode, snapshot, prior,
collectedBlocks, removedINodes, countDiffChange)); collectedBlocks, removedINodes));
} }
} }
} }

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java
View File

@ -20,7 +20,6 @@
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.namenode.INode; import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo; import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
@ -48,8 +47,7 @@ public void destroyAndCollectSnapshotBlocks(
} }
public void saveSelf2Snapshot(int latestSnapshotId, INodeFile iNodeFile, public void saveSelf2Snapshot(int latestSnapshotId, INodeFile iNodeFile,
INodeFileAttributes snapshotCopy, boolean withBlocks) INodeFileAttributes snapshotCopy, boolean withBlocks) {
throws QuotaExceededException {
final FileDiff diff = final FileDiff diff =
super.saveSelf2Snapshot(latestSnapshotId, iNodeFile, snapshotCopy); super.saveSelf2Snapshot(latestSnapshotId, iNodeFile, snapshotCopy);
if(withBlocks) // Store blocks if this is the first update if(withBlocks) // Store blocks if this is the first update

6
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
View File

@ -20,7 +20,6 @@
import java.util.List; import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.namenode.AclFeature; import org.apache.hadoop.hdfs.server.namenode.AclFeature;
import org.apache.hadoop.hdfs.server.namenode.INode; import org.apache.hadoop.hdfs.server.namenode.INode;
@ -117,8 +116,7 @@ public String getDetailedString() {
public Quota.Counts cleanFile(final INodeFile file, final int snapshotId, public Quota.Counts cleanFile(final INodeFile file, final int snapshotId,
int priorSnapshotId, final BlocksMapUpdateInfo collectedBlocks, int priorSnapshotId, final BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes, final boolean countDiffChange) final List<INode> removedINodes) {
throws QuotaExceededException {
if (snapshotId == Snapshot.CURRENT_STATE_ID) { if (snapshotId == Snapshot.CURRENT_STATE_ID) {
// delete the current file while the file has snapshot feature // delete the current file while the file has snapshot feature
if (!isCurrentFileDeleted()) { if (!isCurrentFileDeleted()) {
@ -130,7 +128,7 @@ public Quota.Counts cleanFile(final INodeFile file, final int snapshotId,
} else { // delete the snapshot } else { // delete the snapshot
priorSnapshotId = getDiffs().updatePrior(snapshotId, priorSnapshotId); priorSnapshotId = getDiffs().updatePrior(snapshotId, priorSnapshotId);
return diffs.deleteSnapshotDiff(snapshotId, priorSnapshotId, file, return diffs.deleteSnapshotDiff(snapshotId, priorSnapshotId, file,
collectedBlocks, removedINodes, countDiffChange); collectedBlocks, removedINodes);
} }
} }

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestAclWithSnapshot.java
View File

@ -38,7 +38,6 @@
import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException; import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
import org.apache.hadoop.hdfs.server.namenode.AclFeature; import org.apache.hadoop.hdfs.server.namenode.AclFeature;
import org.apache.hadoop.hdfs.server.namenode.AclStorage; import org.apache.hadoop.hdfs.server.namenode.AclStorage;
@ -611,7 +610,6 @@ public void testChangeAclExceedsQuota() throws Exception {
aclSpec = Lists.newArrayList( aclSpec = Lists.newArrayList(
aclEntry(ACCESS, USER, "bruce", READ)); aclEntry(ACCESS, USER, "bruce", READ));
exception.expect(NSQuotaExceededException.class);
hdfs.modifyAclEntries(filePath, aclSpec); hdfs.modifyAclEntries(filePath, aclSpec);
} }
@ -647,7 +645,6 @@ public void testRemoveAclExceedsQuota() throws Exception {
aclSpec = Lists.newArrayList( aclSpec = Lists.newArrayList(
aclEntry(ACCESS, USER, "bruce", READ)); aclEntry(ACCESS, USER, "bruce", READ));
exception.expect(NSQuotaExceededException.class);
hdfs.removeAcl(filePath); hdfs.removeAcl(filePath);
} }

41
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestNestedSnapshots.java
View File

@ -34,13 +34,11 @@
import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
import org.apache.hadoop.hdfs.protocol.SnapshotException; import org.apache.hadoop.hdfs.protocol.SnapshotException;
import org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream; import org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory; import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
import org.apache.hadoop.hdfs.server.namenode.INode; import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
import org.apache.hadoop.ipc.RemoteException;
import org.junit.After; import org.junit.After;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
@ -239,7 +237,7 @@ public void testSnapshotLimit() throws Exception {
} }
@Test (timeout=300000) @Test (timeout=300000)
public void testSnapshotWithQuota() throws Exception { public void testSnapshotName() throws Exception {
final String dirStr = "/testSnapshotWithQuota/dir"; final String dirStr = "/testSnapshotWithQuota/dir";
final Path dir = new Path(dirStr); final Path dir = new Path(dirStr);
hdfs.mkdirs(dir, new FsPermission((short)0777)); hdfs.mkdirs(dir, new FsPermission((short)0777));
@ -266,43 +264,6 @@ public void testSnapshotWithQuota() throws Exception {
Assert.assertEquals(HdfsConstants.DOT_SNAPSHOT_DIR, parent.getName()); Assert.assertEquals(HdfsConstants.DOT_SNAPSHOT_DIR, parent.getName());
Assert.assertEquals(dir, parent.getParent()); Assert.assertEquals(dir, parent.getParent());
} }
final Path f2 = new Path(foo, "f2");
DFSTestUtil.createFile(hdfs, f2, BLOCKSIZE, REPLICATION, SEED);
try {
// normal create file should fail with quota
final Path f3 = new Path(foo, "f3");
DFSTestUtil.createFile(hdfs, f3, BLOCKSIZE, REPLICATION, SEED);
Assert.fail();
} catch(NSQuotaExceededException e) {
SnapshotTestHelper.LOG.info("The exception is expected.", e);
}
try {
// createSnapshot should fail with quota
hdfs.createSnapshot(dir);
Assert.fail();
} catch(NSQuotaExceededException e) {
SnapshotTestHelper.LOG.info("The exception is expected.", e);
}
try {
// setPermission f1 should fail with quote since it cannot add diff.
hdfs.setPermission(f1, new FsPermission((short)0));
Assert.fail();
} catch(RemoteException e) {
Assert.assertSame(NSQuotaExceededException.class,
e.unwrapRemoteException().getClass());
SnapshotTestHelper.LOG.info("The exception is expected.", e);
}
// setPermission f2 since it was created after the snapshot
hdfs.setPermission(f2, new FsPermission((short)0));
// increase quota and retry the commands.
hdfs.setQuota(dir, NS_QUOTA + 2, HdfsConstants.QUOTA_DONT_SET);
hdfs.createSnapshot(dir, "s1");
hdfs.setPermission(foo, new FsPermission((short)0444));
} }
/** /**

74
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
View File

@ -50,7 +50,7 @@
import org.apache.hadoop.hdfs.client.HdfsDataOutputStream.SyncFlag; import org.apache.hadoop.hdfs.client.HdfsDataOutputStream.SyncFlag;
import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException; import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
@ -62,6 +62,7 @@
import org.apache.hadoop.hdfs.server.namenode.INodeFile; import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.hdfs.server.namenode.INodeReference; import org.apache.hadoop.hdfs.server.namenode.INodeReference;
import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithCount; import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithCount;
import org.apache.hadoop.hdfs.server.namenode.INodesInPath;
import org.apache.hadoop.hdfs.server.namenode.Quota; import org.apache.hadoop.hdfs.server.namenode.Quota;
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.ChildrenDiff; import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.ChildrenDiff;
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiff; import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiff;
@ -73,6 +74,7 @@
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.mockito.Mockito; import org.mockito.Mockito;
import org.mockito.internal.util.reflection.Whitebox;
/** Testing rename with snapshots. */ /** Testing rename with snapshots. */
public class TestRenameWithSnapshots { public class TestRenameWithSnapshots {
@ -1197,7 +1199,7 @@ public void testRenameDirAndDeleteSnapshot_2() throws Exception {
restartClusterAndCheckImage(true); restartClusterAndCheckImage(true);
// make sure the whole referred subtree has been destroyed // make sure the whole referred subtree has been destroyed
Quota.Counts q = fsdir.getRoot().getDirectoryWithQuotaFeature().getSpaceConsumed(); Quota.Counts q = fsdir.getRoot().getDirectoryWithQuotaFeature().getSpaceConsumed();
assertEquals(4, q.get(Quota.NAMESPACE)); assertEquals(3, q.get(Quota.NAMESPACE));
assertEquals(0, q.get(Quota.DISKSPACE)); assertEquals(0, q.get(Quota.DISKSPACE));
hdfs.deleteSnapshot(sdir1, "s1"); hdfs.deleteSnapshot(sdir1, "s1");
@ -1558,29 +1560,34 @@ public void testRenameUndo_5() throws Exception {
SnapshotTestHelper.createSnapshot(hdfs, dir1, "s1"); SnapshotTestHelper.createSnapshot(hdfs, dir1, "s1");
SnapshotTestHelper.createSnapshot(hdfs, dir2, "s2"); SnapshotTestHelper.createSnapshot(hdfs, dir2, "s2");
// set ns quota of dir2 to 5, so the current remaining is 2 (already has // set ns quota of dir2 to 4, so the current remaining is 2 (already has
// dir2, subdir2, and s2) // dir2, and subdir2)
hdfs.setQuota(dir2, 5, Long.MAX_VALUE - 1); hdfs.setQuota(dir2, 4, Long.MAX_VALUE - 1);
final Path foo2 = new Path(subdir2, foo.getName()); final Path foo2 = new Path(subdir2, foo.getName());
FSDirectory fsdir2 = Mockito.spy(fsdir);
Mockito.doThrow(new NSQuotaExceededException("fake exception")).when(fsdir2)
.addLastINode((INodesInPath) Mockito.anyObject(),
(INode) Mockito.anyObject(), Mockito.anyBoolean());
Whitebox.setInternalState(fsn, "dir", fsdir2);
// rename /test/dir1/foo to /test/dir2/subdir2/foo. // rename /test/dir1/foo to /test/dir2/subdir2/foo.
// FSDirectory#verifyQuota4Rename will pass since foo/bar only be counted // FSDirectory#verifyQuota4Rename will pass since the remaining quota is 2.
// as 2 in NS quota. However, the rename operation will fail when adding // However, the rename operation will fail since we let addLastINode throw
// foo to subdir2, since we will create a snapshot diff for subdir2. // NSQuotaExceededException
boolean rename = hdfs.rename(foo, foo2); boolean rename = hdfs.rename(foo, foo2);
assertFalse(rename); assertFalse(rename);
// check the undo // check the undo
assertTrue(hdfs.exists(foo)); assertTrue(hdfs.exists(foo));
assertTrue(hdfs.exists(bar)); assertTrue(hdfs.exists(bar));
INodeDirectory dir1Node = fsdir.getINode4Write(dir1.toString()) INodeDirectory dir1Node = fsdir2.getINode4Write(dir1.toString())
.asDirectory(); .asDirectory();
List<INode> childrenList = ReadOnlyList.Util.asList(dir1Node List<INode> childrenList = ReadOnlyList.Util.asList(dir1Node
.getChildrenList(Snapshot.CURRENT_STATE_ID)); .getChildrenList(Snapshot.CURRENT_STATE_ID));
assertEquals(1, childrenList.size()); assertEquals(1, childrenList.size());
INode fooNode = childrenList.get(0); INode fooNode = childrenList.get(0);
assertTrue(fooNode.asDirectory().isWithSnapshot()); assertTrue(fooNode.asDirectory().isWithSnapshot());
INode barNode = fsdir.getINode4Write(bar.toString()); INode barNode = fsdir2.getINode4Write(bar.toString());
assertTrue(barNode.getClass() == INodeFile.class); assertTrue(barNode.getClass() == INodeFile.class);
assertSame(fooNode, barNode.getParent()); assertSame(fooNode, barNode.getParent());
List<DirectoryDiff> diffList = dir1Node List<DirectoryDiff> diffList = dir1Node
@ -1591,17 +1598,17 @@ public void testRenameUndo_5() throws Exception {
assertTrue(diff.getChildrenDiff().getList(ListType.DELETED).isEmpty()); assertTrue(diff.getChildrenDiff().getList(ListType.DELETED).isEmpty());
// check dir2 // check dir2
INodeDirectory dir2Node = fsdir.getINode4Write(dir2.toString()).asDirectory(); INodeDirectory dir2Node = fsdir2.getINode4Write(dir2.toString()).asDirectory();
assertTrue(dir2Node.isSnapshottable()); assertTrue(dir2Node.isSnapshottable());
Quota.Counts counts = dir2Node.computeQuotaUsage(); Quota.Counts counts = dir2Node.computeQuotaUsage();
assertEquals(3, counts.get(Quota.NAMESPACE)); assertEquals(2, counts.get(Quota.NAMESPACE));
assertEquals(0, counts.get(Quota.DISKSPACE)); assertEquals(0, counts.get(Quota.DISKSPACE));
childrenList = ReadOnlyList.Util.asList(dir2Node.asDirectory() childrenList = ReadOnlyList.Util.asList(dir2Node.asDirectory()
.getChildrenList(Snapshot.CURRENT_STATE_ID)); .getChildrenList(Snapshot.CURRENT_STATE_ID));
assertEquals(1, childrenList.size()); assertEquals(1, childrenList.size());
INode subdir2Node = childrenList.get(0); INode subdir2Node = childrenList.get(0);
assertSame(dir2Node, subdir2Node.getParent()); assertSame(dir2Node, subdir2Node.getParent());
assertSame(subdir2Node, fsdir.getINode4Write(subdir2.toString())); assertSame(subdir2Node, fsdir2.getINode4Write(subdir2.toString()));
diffList = dir2Node.getDiffs().asList(); diffList = dir2Node.getDiffs().asList();
assertEquals(1, diffList.size()); assertEquals(1, diffList.size());
diff = diffList.get(0); diff = diffList.get(0);
@ -1628,27 +1635,28 @@ public void testRenameUndo_6() throws Exception {
SnapshotTestHelper.createSnapshot(hdfs, dir1, "s1"); SnapshotTestHelper.createSnapshot(hdfs, dir1, "s1");
SnapshotTestHelper.createSnapshot(hdfs, dir2, "s2"); SnapshotTestHelper.createSnapshot(hdfs, dir2, "s2");
// set ns quota of dir2 to 4, so the current remaining is 0 (already has // set ns quota of dir2 to 4, so the current remaining is 1 (already has
// dir2, sub_dir2, subsub_dir2, and s2) // dir2, sub_dir2, and subsub_dir2)
hdfs.setQuota(dir2, 4, Long.MAX_VALUE - 1); hdfs.setQuota(dir2, 4, Long.MAX_VALUE - 1);
FSDirectory fsdir2 = Mockito.spy(fsdir);
Mockito.doThrow(new RuntimeException("fake exception")).when(fsdir2)
.removeLastINode((INodesInPath) Mockito.anyObject());
Whitebox.setInternalState(fsn, "dir", fsdir2);
// rename /test/dir1/foo to /test/dir2/sub_dir2/subsub_dir2. // rename /test/dir1/foo to /test/dir2/sub_dir2/subsub_dir2.
// FSDirectory#verifyQuota4Rename will pass since foo only be counted // FSDirectory#verifyQuota4Rename will pass since foo only be counted
// as 1 in NS quota. However, the rename operation will fail when removing // as 1 in NS quota. However, the rename operation will fail when removing
// subsub_dir2 since this step tries to add a snapshot diff in sub_dir2. // subsub_dir2.
try { try {
hdfs.rename(foo, subsub_dir2, Rename.OVERWRITE); hdfs.rename(foo, subsub_dir2, Rename.OVERWRITE);
fail("Expect QuotaExceedException"); fail("Expect QuotaExceedException");
} catch (QuotaExceededException e) { } catch (Exception e) {
String msg = "Failed to record modification for snapshot: " String msg = "fake exception";
+ "The NameSpace quota (directories and files)"
+ " is exceeded: quota=4 file count=5";
GenericTestUtils.assertExceptionContains(msg, e); GenericTestUtils.assertExceptionContains(msg, e);
} }
// check the undo // check the undo
assertTrue(hdfs.exists(foo)); assertTrue(hdfs.exists(foo));
INodeDirectory dir1Node = fsdir.getINode4Write(dir1.toString()) INodeDirectory dir1Node = fsdir2.getINode4Write(dir1.toString())
.asDirectory(); .asDirectory();
List<INode> childrenList = ReadOnlyList.Util.asList(dir1Node List<INode> childrenList = ReadOnlyList.Util.asList(dir1Node
.getChildrenList(Snapshot.CURRENT_STATE_ID)); .getChildrenList(Snapshot.CURRENT_STATE_ID));
@ -1664,19 +1672,18 @@ public void testRenameUndo_6() throws Exception {
assertTrue(diff.getChildrenDiff().getList(ListType.DELETED).isEmpty()); assertTrue(diff.getChildrenDiff().getList(ListType.DELETED).isEmpty());
// check dir2 // check dir2
INodeDirectory dir2Node = fsdir.getINode4Write(dir2.toString()).asDirectory(); INodeDirectory dir2Node = fsdir2.getINode4Write(dir2.toString()).asDirectory();
assertTrue(dir2Node.isSnapshottable()); assertTrue(dir2Node.isSnapshottable());
Quota.Counts counts = dir2Node.computeQuotaUsage(); Quota.Counts counts = dir2Node.computeQuotaUsage();
assertEquals(4, counts.get(Quota.NAMESPACE)); assertEquals(3, counts.get(Quota.NAMESPACE));
assertEquals(0, counts.get(Quota.DISKSPACE)); assertEquals(0, counts.get(Quota.DISKSPACE));
childrenList = ReadOnlyList.Util.asList(dir2Node.asDirectory() childrenList = ReadOnlyList.Util.asList(dir2Node.asDirectory()
.getChildrenList(Snapshot.CURRENT_STATE_ID)); .getChildrenList(Snapshot.CURRENT_STATE_ID));
assertEquals(1, childrenList.size()); assertEquals(1, childrenList.size());
INode subdir2Node = childrenList.get(0); INode subdir2Node = childrenList.get(0);
assertTrue(subdir2Node.asDirectory().isWithSnapshot());
assertSame(dir2Node, subdir2Node.getParent()); assertSame(dir2Node, subdir2Node.getParent());
assertSame(subdir2Node, fsdir.getINode4Write(sub_dir2.toString())); assertSame(subdir2Node, fsdir2.getINode4Write(sub_dir2.toString()));
INode subsubdir2Node = fsdir.getINode4Write(subsub_dir2.toString()); INode subsubdir2Node = fsdir2.getINode4Write(subsub_dir2.toString());
assertTrue(subsubdir2Node.getClass() == INodeDirectory.class); assertTrue(subsubdir2Node.getClass() == INodeDirectory.class);
assertSame(subdir2Node, subsubdir2Node.getParent()); assertSame(subdir2Node, subsubdir2Node.getParent());
@ -1685,9 +1692,6 @@ public void testRenameUndo_6() throws Exception {
diff = diffList.get(0); diff = diffList.get(0);
assertTrue(diff.getChildrenDiff().getList(ListType.CREATED).isEmpty()); assertTrue(diff.getChildrenDiff().getList(ListType.CREATED).isEmpty());
assertTrue(diff.getChildrenDiff().getList(ListType.DELETED).isEmpty()); assertTrue(diff.getChildrenDiff().getList(ListType.DELETED).isEmpty());
diffList = subdir2Node.asDirectory().getDiffs().asList();
assertEquals(0, diffList.size());
} }
/** /**
@ -1787,7 +1791,7 @@ public void testRenameExceedQuota() throws Exception {
INode dir2Node = fsdir.getINode4Write(dir2.toString()); INode dir2Node = fsdir.getINode4Write(dir2.toString());
assertTrue(dir2Node.asDirectory().isSnapshottable()); assertTrue(dir2Node.asDirectory().isSnapshottable());
Quota.Counts counts = dir2Node.computeQuotaUsage(); Quota.Counts counts = dir2Node.computeQuotaUsage();
assertEquals(7, counts.get(Quota.NAMESPACE)); assertEquals(4, counts.get(Quota.NAMESPACE));
assertEquals(BLOCKSIZE * REPL * 2, counts.get(Quota.DISKSPACE)); assertEquals(BLOCKSIZE * REPL * 2, counts.get(Quota.DISKSPACE));
} }
@ -1955,11 +1959,11 @@ public void testRenameDirAndDeleteSnapshot_3() throws Exception {
final INodeDirectory dir1Node = fsdir.getINode4Write(sdir1.toString()) final INodeDirectory dir1Node = fsdir.getINode4Write(sdir1.toString())
.asDirectory(); .asDirectory();
Quota.Counts q1 = dir1Node.getDirectoryWithQuotaFeature().getSpaceConsumed(); Quota.Counts q1 = dir1Node.getDirectoryWithQuotaFeature().getSpaceConsumed();
assertEquals(4, q1.get(Quota.NAMESPACE)); assertEquals(3, q1.get(Quota.NAMESPACE));
final INodeDirectory dir2Node = fsdir.getINode4Write(sdir2.toString()) final INodeDirectory dir2Node = fsdir.getINode4Write(sdir2.toString())
.asDirectory(); .asDirectory();
Quota.Counts q2 = dir2Node.getDirectoryWithQuotaFeature().getSpaceConsumed(); Quota.Counts q2 = dir2Node.getDirectoryWithQuotaFeature().getSpaceConsumed();
assertEquals(2, q2.get(Quota.NAMESPACE)); assertEquals(1, q2.get(Quota.NAMESPACE));
final Path foo_s1 = SnapshotTestHelper.getSnapshotPath(sdir1, "s1", final Path foo_s1 = SnapshotTestHelper.getSnapshotPath(sdir1, "s1",
foo.getName()); foo.getName());
@ -2025,11 +2029,11 @@ public void testRenameDirAndDeleteSnapshot_4() throws Exception {
.asDirectory(); .asDirectory();
// sdir1 + s1 + foo_s1 (foo) + foo (foo + s1 + bar~bar3) // sdir1 + s1 + foo_s1 (foo) + foo (foo + s1 + bar~bar3)
Quota.Counts q1 = dir1Node.getDirectoryWithQuotaFeature().getSpaceConsumed(); Quota.Counts q1 = dir1Node.getDirectoryWithQuotaFeature().getSpaceConsumed();
assertEquals(9, q1.get(Quota.NAMESPACE)); assertEquals(7, q1.get(Quota.NAMESPACE));
final INodeDirectory dir2Node = fsdir.getINode4Write(sdir2.toString()) final INodeDirectory dir2Node = fsdir.getINode4Write(sdir2.toString())
.asDirectory(); .asDirectory();
Quota.Counts q2 = dir2Node.getDirectoryWithQuotaFeature().getSpaceConsumed(); Quota.Counts q2 = dir2Node.getDirectoryWithQuotaFeature().getSpaceConsumed();
assertEquals(2, q2.get(Quota.NAMESPACE)); assertEquals(1, q2.get(Quota.NAMESPACE));
final Path foo_s1 = SnapshotTestHelper.getSnapshotPath(sdir1, "s1", final Path foo_s1 = SnapshotTestHelper.getSnapshotPath(sdir1, "s1",
foo.getName()); foo.getName());

119
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
View File

@ -263,7 +263,7 @@ public void testDeleteCurrentFileDirectory() throws Exception {
BlockInfo[] blocks = temp.getBlocks(); BlockInfo[] blocks = temp.getBlocks();
hdfs.delete(tempDir, true); hdfs.delete(tempDir, true);
// check dir's quota usage // check dir's quota usage
checkQuotaUsageComputation(dir, 9L, BLOCKSIZE * REPLICATION * 3); checkQuotaUsageComputation(dir, 8, BLOCKSIZE * REPLICATION * 3);
// check blocks of tempFile // check blocks of tempFile
for (BlockInfo b : blocks) { for (BlockInfo b : blocks) {
assertNull(blockmanager.getBlockCollection(b)); assertNull(blockmanager.getBlockCollection(b));
@ -279,7 +279,7 @@ public void testDeleteCurrentFileDirectory() throws Exception {
// create snapshot s1 // create snapshot s1
SnapshotTestHelper.createSnapshot(hdfs, dir, "s1"); SnapshotTestHelper.createSnapshot(hdfs, dir, "s1");
// check dir's quota usage // check dir's quota usage
checkQuotaUsageComputation(dir, 14L, BLOCKSIZE * REPLICATION * 4); checkQuotaUsageComputation(dir, 9L, BLOCKSIZE * REPLICATION * 4);
// get two snapshots for later use // get two snapshots for later use
Snapshot snapshot0 = fsdir.getINode(dir.toString()).asDirectory() Snapshot snapshot0 = fsdir.getINode(dir.toString()).asDirectory()
@ -295,7 +295,7 @@ public void testDeleteCurrentFileDirectory() throws Exception {
hdfs.delete(noChangeDirParent, true); hdfs.delete(noChangeDirParent, true);
// while deletion, we add a diff for metaChangeFile2 as its snapshot copy // while deletion, we add a diff for metaChangeFile2 as its snapshot copy
// for s1, we also add diffs for both sub and noChangeDirParent // for s1, we also add diffs for both sub and noChangeDirParent
checkQuotaUsageComputation(dir, 17L, BLOCKSIZE * REPLICATION * 4); checkQuotaUsageComputation(dir, 9L, BLOCKSIZE * REPLICATION * 4);
// check the snapshot copy of noChangeDir // check the snapshot copy of noChangeDir
Path snapshotNoChangeDir = SnapshotTestHelper.getSnapshotPath(dir, "s1", Path snapshotNoChangeDir = SnapshotTestHelper.getSnapshotPath(dir, "s1",
@ -337,11 +337,11 @@ public void testDeleteCurrentFileDirectory() throws Exception {
final INodeFile newFileNode = TestSnapshotBlocksMap.assertBlockCollection( final INodeFile newFileNode = TestSnapshotBlocksMap.assertBlockCollection(
newFile.toString(), 1, fsdir, blockmanager); newFile.toString(), 1, fsdir, blockmanager);
blocks = newFileNode.getBlocks(); blocks = newFileNode.getBlocks();
checkQuotaUsageComputation(dir, 18L, BLOCKSIZE * REPLICATION * 5); checkQuotaUsageComputation(dir, 10L, BLOCKSIZE * REPLICATION * 5);
hdfs.delete(sub, true); hdfs.delete(sub, true);
// while deletion, we add diff for subsub and metaChangeFile1, and remove // while deletion, we add diff for subsub and metaChangeFile1, and remove
// newFile // newFile
checkQuotaUsageComputation(dir, 19L, BLOCKSIZE * REPLICATION * 4); checkQuotaUsageComputation(dir, 9L, BLOCKSIZE * REPLICATION * 4);
for (BlockInfo b : blocks) { for (BlockInfo b : blocks) {
assertNull(blockmanager.getBlockCollection(b)); assertNull(blockmanager.getBlockCollection(b));
} }
@ -426,13 +426,13 @@ public void testDeleteEarliestSnapshot1() throws Exception {
// create snapshot s1 for sub // create snapshot s1 for sub
SnapshotTestHelper.createSnapshot(hdfs, sub, snapshotName); SnapshotTestHelper.createSnapshot(hdfs, sub, snapshotName);
// check quota usage computation // check quota usage computation
checkQuotaUsageComputation(sub, 4, BLOCKSIZE * REPLICATION * 2); checkQuotaUsageComputation(sub, 3, BLOCKSIZE * REPLICATION * 2);
// delete s1 // delete s1
hdfs.deleteSnapshot(sub, snapshotName); hdfs.deleteSnapshot(sub, snapshotName);
checkQuotaUsageComputation(sub, 3, BLOCKSIZE * REPLICATION * 2); checkQuotaUsageComputation(sub, 3, BLOCKSIZE * REPLICATION * 2);
// now we can create a snapshot with the same name // now we can create a snapshot with the same name
hdfs.createSnapshot(sub, snapshotName); hdfs.createSnapshot(sub, snapshotName);
checkQuotaUsageComputation(sub, 4, BLOCKSIZE * REPLICATION * 2); checkQuotaUsageComputation(sub, 3, BLOCKSIZE * REPLICATION * 2);
// create a new file under sub // create a new file under sub
Path newFile = new Path(sub, "newFile"); Path newFile = new Path(sub, "newFile");
@ -440,14 +440,14 @@ public void testDeleteEarliestSnapshot1() throws Exception {
// create another snapshot s2 // create another snapshot s2
String snapshotName2 = "s2"; String snapshotName2 = "s2";
hdfs.createSnapshot(sub, snapshotName2); hdfs.createSnapshot(sub, snapshotName2);
checkQuotaUsageComputation(sub, 6, BLOCKSIZE * REPLICATION * 3); checkQuotaUsageComputation(sub, 4, BLOCKSIZE * REPLICATION * 3);
// Get the filestatus of sub under snapshot s2 // Get the filestatus of sub under snapshot s2
Path ss = SnapshotTestHelper Path ss = SnapshotTestHelper
.getSnapshotPath(sub, snapshotName2, "newFile"); .getSnapshotPath(sub, snapshotName2, "newFile");
FileStatus statusBeforeDeletion = hdfs.getFileStatus(ss); FileStatus statusBeforeDeletion = hdfs.getFileStatus(ss);
// delete s1 // delete s1
hdfs.deleteSnapshot(sub, snapshotName); hdfs.deleteSnapshot(sub, snapshotName);
checkQuotaUsageComputation(sub, 5, BLOCKSIZE * REPLICATION * 3); checkQuotaUsageComputation(sub, 4, BLOCKSIZE * REPLICATION * 3);
FileStatus statusAfterDeletion = hdfs.getFileStatus(ss); FileStatus statusAfterDeletion = hdfs.getFileStatus(ss);
System.out.println("Before deletion: " + statusBeforeDeletion.toString() System.out.println("Before deletion: " + statusBeforeDeletion.toString()
+ "\n" + "After deletion: " + statusAfterDeletion.toString()); + "\n" + "After deletion: " + statusAfterDeletion.toString());
@ -483,28 +483,28 @@ public void testDeleteEarliestSnapshot2() throws Exception {
// create snapshot s0 on dir // create snapshot s0 on dir
SnapshotTestHelper.createSnapshot(hdfs, dir, "s0"); SnapshotTestHelper.createSnapshot(hdfs, dir, "s0");
checkQuotaUsageComputation(dir, 8, 3 * BLOCKSIZE * REPLICATION); checkQuotaUsageComputation(dir, 7, 3 * BLOCKSIZE * REPLICATION);
// delete /TestSnapshot/sub/noChangeDir/metaChangeDir/toDeleteFile // delete /TestSnapshot/sub/noChangeDir/metaChangeDir/toDeleteFile
hdfs.delete(toDeleteFile, true); hdfs.delete(toDeleteFile, true);
// the deletion adds diff of toDeleteFile and metaChangeDir // the deletion adds diff of toDeleteFile and metaChangeDir
checkQuotaUsageComputation(dir, 10, 3 * BLOCKSIZE * REPLICATION); checkQuotaUsageComputation(dir, 7, 3 * BLOCKSIZE * REPLICATION);
// change metadata of /TestSnapshot/sub/noChangeDir/metaChangeDir and // change metadata of /TestSnapshot/sub/noChangeDir/metaChangeDir and
// /TestSnapshot/sub/noChangeDir/metaChangeFile // /TestSnapshot/sub/noChangeDir/metaChangeFile
hdfs.setReplication(metaChangeFile, REPLICATION_1); hdfs.setReplication(metaChangeFile, REPLICATION_1);
hdfs.setOwner(metaChangeDir, "unknown", "unknown"); hdfs.setOwner(metaChangeDir, "unknown", "unknown");
checkQuotaUsageComputation(dir, 11, 3 * BLOCKSIZE * REPLICATION); checkQuotaUsageComputation(dir, 7, 3 * BLOCKSIZE * REPLICATION);
// create snapshot s1 on dir // create snapshot s1 on dir
hdfs.createSnapshot(dir, "s1"); hdfs.createSnapshot(dir, "s1");
checkQuotaUsageComputation(dir, 12, 3 * BLOCKSIZE * REPLICATION); checkQuotaUsageComputation(dir, 7, 3 * BLOCKSIZE * REPLICATION);
// delete snapshot s0 // delete snapshot s0
hdfs.deleteSnapshot(dir, "s0"); hdfs.deleteSnapshot(dir, "s0");
// namespace: remove toDeleteFile and its diff, metaChangeFile's diff, // namespace: remove toDeleteFile and its diff, metaChangeFile's diff,
// metaChangeDir's diff, dir's diff. diskspace: remove toDeleteFile, and // metaChangeDir's diff, dir's diff. diskspace: remove toDeleteFile, and
// metaChangeFile's replication factor decreases // metaChangeFile's replication factor decreases
checkQuotaUsageComputation(dir, 7, 2 * BLOCKSIZE * REPLICATION - BLOCKSIZE); checkQuotaUsageComputation(dir, 6, 2 * BLOCKSIZE * REPLICATION - BLOCKSIZE);
for (BlockInfo b : blocks) { for (BlockInfo b : blocks) {
assertNull(blockmanager.getBlockCollection(b)); assertNull(blockmanager.getBlockCollection(b));
} }
@ -679,21 +679,21 @@ public void testCombineSnapshotDiff3() throws Exception {
// create another snapshot // create another snapshot
SnapshotTestHelper.createSnapshot(hdfs, dir, "s2"); SnapshotTestHelper.createSnapshot(hdfs, dir, "s2");
checkQuotaUsageComputation(dir, 11, BLOCKSIZE * 2 * REPLICATION); checkQuotaUsageComputation(dir, 7, BLOCKSIZE * 2 * REPLICATION);
// delete subsubdir and subDir2 // delete subsubdir and subDir2
hdfs.delete(subsubDir, true); hdfs.delete(subsubDir, true);
hdfs.delete(subDir2, true); hdfs.delete(subDir2, true);
// add diff of s2 to subDir1, subsubDir, and subDir2 // add diff of s2 to subDir1, subsubDir, and subDir2
checkQuotaUsageComputation(dir, 14, BLOCKSIZE * 2 * REPLICATION); checkQuotaUsageComputation(dir, 7, BLOCKSIZE * 2 * REPLICATION);
// delete snapshot s2 // delete snapshot s2
hdfs.deleteSnapshot(dir, "s2"); hdfs.deleteSnapshot(dir, "s2");
// delete s2 diff in dir, subDir2, and subsubDir. Delete newFile, newDir, // delete s2 diff in dir, subDir2, and subsubDir. Delete newFile, newDir,
// and newFile2. Rename s2 diff to s1 for subDir1 // and newFile2. Rename s2 diff to s1 for subDir1
checkQuotaUsageComputation(dir, 8, 0); checkQuotaUsageComputation(dir, 4, 0);
// Check rename of snapshot diff in subDir1 // Check rename of snapshot diff in subDir1
Path subdir1_s1 = SnapshotTestHelper.getSnapshotPath(dir, "s1", Path subdir1_s1 = SnapshotTestHelper.getSnapshotPath(dir, "s1",
subDir1.getName()); subDir1.getName());
@ -714,7 +714,6 @@ private void testCombineSnapshotDiffImpl(Path snapshotRoot, String modDirStr,
int dirNodeNum) throws Exception { int dirNodeNum) throws Exception {
Path modDir = modDirStr.isEmpty() ? snapshotRoot : new Path(snapshotRoot, Path modDir = modDirStr.isEmpty() ? snapshotRoot : new Path(snapshotRoot,
modDirStr); modDirStr);
final int delta = modDirStr.isEmpty() ? 0 : 1;
Path file10 = new Path(modDir, "file10"); Path file10 = new Path(modDir, "file10");
Path file11 = new Path(modDir, "file11"); Path file11 = new Path(modDir, "file11");
Path file12 = new Path(modDir, "file12"); Path file12 = new Path(modDir, "file12");
@ -728,72 +727,59 @@ private void testCombineSnapshotDiffImpl(Path snapshotRoot, String modDirStr,
// create snapshot s1 for snapshotRoot // create snapshot s1 for snapshotRoot
SnapshotTestHelper.createSnapshot(hdfs, snapshotRoot, "s1"); SnapshotTestHelper.createSnapshot(hdfs, snapshotRoot, "s1");
checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 5, 8 * BLOCKSIZE); checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 4, 8 * BLOCKSIZE);
// delete file11 // delete file11
hdfs.delete(file11, true); hdfs.delete(file11, true);
checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 6 + delta, checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 4, 8 * BLOCKSIZE);
8 * BLOCKSIZE);
// modify file12 // modify file12
hdfs.setReplication(file12, REPLICATION); hdfs.setReplication(file12, REPLICATION);
checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 7 + delta, checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 4, 9 * BLOCKSIZE);
9 * BLOCKSIZE);
// modify file13 // modify file13
hdfs.setReplication(file13, REPLICATION); hdfs.setReplication(file13, REPLICATION);
checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 8 + delta, checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 4, 10 * BLOCKSIZE);
10 * BLOCKSIZE);
// create file14 // create file14
DFSTestUtil.createFile(hdfs, file14, BLOCKSIZE, REPLICATION, seed); DFSTestUtil.createFile(hdfs, file14, BLOCKSIZE, REPLICATION, seed);
checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 9 + delta, checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 5, 13 * BLOCKSIZE);
13 * BLOCKSIZE);
// create file15 // create file15
DFSTestUtil.createFile(hdfs, file15, BLOCKSIZE, REPLICATION, seed); DFSTestUtil.createFile(hdfs, file15, BLOCKSIZE, REPLICATION, seed);
checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 10 + delta, checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 6, 16 * BLOCKSIZE);
16 * BLOCKSIZE);
// create snapshot s2 for snapshotRoot // create snapshot s2 for snapshotRoot
hdfs.createSnapshot(snapshotRoot, "s2"); hdfs.createSnapshot(snapshotRoot, "s2");
checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 11 + delta, checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 6, 16 * BLOCKSIZE);
16 * BLOCKSIZE);
// create file11 again: (0, d) + (c, 0) // create file11 again: (0, d) + (c, 0)
DFSTestUtil.createFile(hdfs, file11, BLOCKSIZE, REPLICATION, seed); DFSTestUtil.createFile(hdfs, file11, BLOCKSIZE, REPLICATION, seed);
checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 12 + delta * 2, checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 7, 19 * BLOCKSIZE);
19 * BLOCKSIZE);
// delete file12 // delete file12
hdfs.delete(file12, true); hdfs.delete(file12, true);
checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 13 + delta * 2, checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 7, 19 * BLOCKSIZE);
19 * BLOCKSIZE);
// modify file13 // modify file13
hdfs.setReplication(file13, (short) (REPLICATION - 2)); hdfs.setReplication(file13, (short) (REPLICATION - 2));
checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 14 + delta * 2, checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 7, 19 * BLOCKSIZE);
19 * BLOCKSIZE);
// delete file14: (c, 0) + (0, d) // delete file14: (c, 0) + (0, d)
hdfs.delete(file14, true); hdfs.delete(file14, true);
checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 15 + delta * 2, checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 7, 19 * BLOCKSIZE);
19 * BLOCKSIZE);
// modify file15 // modify file15
hdfs.setReplication(file15, REPLICATION_1); hdfs.setReplication(file15, REPLICATION_1);
checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 16 + delta * 2, checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 7, 19 * BLOCKSIZE);
19 * BLOCKSIZE);
// create snapshot s3 for snapshotRoot // create snapshot s3 for snapshotRoot
hdfs.createSnapshot(snapshotRoot, "s3"); hdfs.createSnapshot(snapshotRoot, "s3");
checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 17 + delta * 2, checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 7, 19 * BLOCKSIZE);
19 * BLOCKSIZE);
// modify file10, to check if the posterior diff was set correctly // modify file10, to check if the posterior diff was set correctly
hdfs.setReplication(file10, REPLICATION); hdfs.setReplication(file10, REPLICATION);
checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 18 + delta * 2, checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 7, 20 * BLOCKSIZE);
20 * BLOCKSIZE);
Path file10_s1 = SnapshotTestHelper.getSnapshotPath(snapshotRoot, "s1", Path file10_s1 = SnapshotTestHelper.getSnapshotPath(snapshotRoot, "s1",
modDirStr + "file10"); modDirStr + "file10");
@ -819,8 +805,7 @@ private void testCombineSnapshotDiffImpl(Path snapshotRoot, String modDirStr,
// delete s2, in which process we need to combine the diff in s2 to s1 // delete s2, in which process we need to combine the diff in s2 to s1
hdfs.deleteSnapshot(snapshotRoot, "s2"); hdfs.deleteSnapshot(snapshotRoot, "s2");
checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 12 + delta, checkQuotaUsageComputation(snapshotRoot, dirNodeNum + 6, 14 * BLOCKSIZE);
14 * BLOCKSIZE);
// check the correctness of s1 // check the correctness of s1
FileStatus statusAfterDeletion10 = hdfs.getFileStatus(file10_s1); FileStatus statusAfterDeletion10 = hdfs.getFileStatus(file10_s1);
@ -873,23 +858,23 @@ public void testDeleteSnapshotWithDirModification() throws Exception {
// create snapshot s1 for sub1, and change the metadata of sub1 // create snapshot s1 for sub1, and change the metadata of sub1
SnapshotTestHelper.createSnapshot(hdfs, sub, "s1"); SnapshotTestHelper.createSnapshot(hdfs, sub, "s1");
checkQuotaUsageComputation(sub, 3, BLOCKSIZE * 3); checkQuotaUsageComputation(sub, 2, BLOCKSIZE * 3);
hdfs.setOwner(sub, "user2", "group2"); hdfs.setOwner(sub, "user2", "group2");
checkQuotaUsageComputation(sub, 3, BLOCKSIZE * 3); checkQuotaUsageComputation(sub, 2, BLOCKSIZE * 3);
// create snapshot s2 for sub1, but do not modify sub1 afterwards // create snapshot s2 for sub1, but do not modify sub1 afterwards
hdfs.createSnapshot(sub, "s2"); hdfs.createSnapshot(sub, "s2");
checkQuotaUsageComputation(sub, 4, BLOCKSIZE * 3); checkQuotaUsageComputation(sub, 2, BLOCKSIZE * 3);
// create snapshot s3 for sub1, and change the metadata of sub1 // create snapshot s3 for sub1, and change the metadata of sub1
hdfs.createSnapshot(sub, "s3"); hdfs.createSnapshot(sub, "s3");
checkQuotaUsageComputation(sub, 5, BLOCKSIZE * 3); checkQuotaUsageComputation(sub, 2, BLOCKSIZE * 3);
hdfs.setOwner(sub, "user3", "group3"); hdfs.setOwner(sub, "user3", "group3");
checkQuotaUsageComputation(sub, 5, BLOCKSIZE * 3); checkQuotaUsageComputation(sub, 2, BLOCKSIZE * 3);
// delete snapshot s3 // delete snapshot s3
hdfs.deleteSnapshot(sub, "s3"); hdfs.deleteSnapshot(sub, "s3");
checkQuotaUsageComputation(sub, 4, BLOCKSIZE * 3); checkQuotaUsageComputation(sub, 2, BLOCKSIZE * 3);
// check sub1's metadata in snapshot s2 // check sub1's metadata in snapshot s2
FileStatus statusOfS2 = hdfs.getFileStatus(new Path(sub, FileStatus statusOfS2 = hdfs.getFileStatus(new Path(sub,
@ -899,7 +884,7 @@ public void testDeleteSnapshotWithDirModification() throws Exception {
// delete snapshot s2 // delete snapshot s2
hdfs.deleteSnapshot(sub, "s2"); hdfs.deleteSnapshot(sub, "s2");
checkQuotaUsageComputation(sub, 3, BLOCKSIZE * 3); checkQuotaUsageComputation(sub, 2, BLOCKSIZE * 3);
// check sub1's metadata in snapshot s1 // check sub1's metadata in snapshot s1
FileStatus statusOfS1 = hdfs.getFileStatus(new Path(sub, FileStatus statusOfS1 = hdfs.getFileStatus(new Path(sub,
@ -957,34 +942,34 @@ public void testRenameSnapshotDiff() throws Exception {
// create snapshot s0 on sub // create snapshot s0 on sub
SnapshotTestHelper.createSnapshot(hdfs, sub, "s0"); SnapshotTestHelper.createSnapshot(hdfs, sub, "s0");
checkQuotaUsageComputation(sub, 5, BLOCKSIZE * 6); checkQuotaUsageComputation(sub, 4, BLOCKSIZE * 6);
// make some changes on both sub and subsub // make some changes on both sub and subsub
final Path subFile1 = new Path(sub, "file1"); final Path subFile1 = new Path(sub, "file1");
final Path subsubFile1 = new Path(subsub, "file1"); final Path subsubFile1 = new Path(subsub, "file1");
DFSTestUtil.createFile(hdfs, subFile1, BLOCKSIZE, REPLICATION_1, seed); DFSTestUtil.createFile(hdfs, subFile1, BLOCKSIZE, REPLICATION_1, seed);
DFSTestUtil.createFile(hdfs, subsubFile1, BLOCKSIZE, REPLICATION, seed); DFSTestUtil.createFile(hdfs, subsubFile1, BLOCKSIZE, REPLICATION, seed);
checkQuotaUsageComputation(sub, 8, BLOCKSIZE * 11); checkQuotaUsageComputation(sub, 6, BLOCKSIZE * 11);
// create snapshot s1 on sub // create snapshot s1 on sub
SnapshotTestHelper.createSnapshot(hdfs, sub, "s1"); SnapshotTestHelper.createSnapshot(hdfs, sub, "s1");
checkQuotaUsageComputation(sub, 9, BLOCKSIZE * 11); checkQuotaUsageComputation(sub, 6, BLOCKSIZE * 11);
// create snapshot s2 on dir // create snapshot s2 on dir
SnapshotTestHelper.createSnapshot(hdfs, dir, "s2"); SnapshotTestHelper.createSnapshot(hdfs, dir, "s2");
checkQuotaUsageComputation(dir, 11, BLOCKSIZE * 11); checkQuotaUsageComputation(dir, 7, BLOCKSIZE * 11);
checkQuotaUsageComputation(sub, 9, BLOCKSIZE * 11); checkQuotaUsageComputation(sub, 6, BLOCKSIZE * 11);
// make changes on subsub and subsubFile1 // make changes on subsub and subsubFile1
hdfs.setOwner(subsub, "unknown", "unknown"); hdfs.setOwner(subsub, "unknown", "unknown");
hdfs.setReplication(subsubFile1, REPLICATION_1); hdfs.setReplication(subsubFile1, REPLICATION_1);
checkQuotaUsageComputation(dir, 13, BLOCKSIZE * 11); checkQuotaUsageComputation(dir, 7, BLOCKSIZE * 11);
checkQuotaUsageComputation(sub, 11, BLOCKSIZE * 11); checkQuotaUsageComputation(sub, 6, BLOCKSIZE * 11);
// make changes on sub // make changes on sub
hdfs.delete(subFile1, true); hdfs.delete(subFile1, true);
checkQuotaUsageComputation(new Path("/"), 16, BLOCKSIZE * 11); checkQuotaUsageComputation(new Path("/"), 8, BLOCKSIZE * 11);
checkQuotaUsageComputation(dir, 15, BLOCKSIZE * 11); checkQuotaUsageComputation(dir, 7, BLOCKSIZE * 11);
checkQuotaUsageComputation(sub, 13, BLOCKSIZE * 11); checkQuotaUsageComputation(sub, 6, BLOCKSIZE * 11);
Path subsubSnapshotCopy = SnapshotTestHelper.getSnapshotPath(dir, "s2", Path subsubSnapshotCopy = SnapshotTestHelper.getSnapshotPath(dir, "s2",
sub.getName() + Path.SEPARATOR + subsub.getName()); sub.getName() + Path.SEPARATOR + subsub.getName());
@ -1003,9 +988,9 @@ public void testRenameSnapshotDiff() throws Exception {
// delete snapshot s2 // delete snapshot s2
hdfs.deleteSnapshot(dir, "s2"); hdfs.deleteSnapshot(dir, "s2");
checkQuotaUsageComputation(new Path("/"), 14, BLOCKSIZE * 11); checkQuotaUsageComputation(new Path("/"), 8, BLOCKSIZE * 11);
checkQuotaUsageComputation(dir, 13, BLOCKSIZE * 11); checkQuotaUsageComputation(dir, 7, BLOCKSIZE * 11);
checkQuotaUsageComputation(sub, 12, BLOCKSIZE * 11); checkQuotaUsageComputation(sub, 6, BLOCKSIZE * 11);
// no snapshot copy for s2 // no snapshot copy for s2
try { try {

71
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestXAttrWithSnapshot.java
View File

@ -343,77 +343,6 @@ public void testRemoveXAttrSnapshotPath() throws Exception {
hdfs.removeXAttr(snapshotPath, name1); hdfs.removeXAttr(snapshotPath, name1);
} }
/**
* Assert exception of setting xattr when exceeding quota.
*/
@Test
public void testSetXAttrExceedsQuota() throws Exception {
Path filePath = new Path(path, "file1");
Path fileSnapshotPath = new Path(snapshotPath, "file1");
FileSystem.mkdirs(hdfs, path, FsPermission.createImmutable((short) 0755));
hdfs.allowSnapshot(path);
hdfs.setQuota(path, 3, HdfsConstants.QUOTA_DONT_SET);
FileSystem.create(hdfs, filePath,
FsPermission.createImmutable((short) 0600)).close();
hdfs.setXAttr(filePath, name1, value1);
hdfs.createSnapshot(path, snapshotName);
byte[] value = hdfs.getXAttr(filePath, name1);
Assert.assertArrayEquals(value, value1);
value = hdfs.getXAttr(fileSnapshotPath, name1);
Assert.assertArrayEquals(value, value1);
exception.expect(NSQuotaExceededException.class);
hdfs.setXAttr(filePath, name2, value2);
}
/**
* Test that an exception is thrown when adding an XAttr Feature to
* a snapshotted path
*/
@Test
public void testSetXAttrAfterSnapshotExceedsQuota() throws Exception {
Path filePath = new Path(path, "file1");
FileSystem.mkdirs(hdfs, path, FsPermission.createImmutable((short) 0755));
hdfs.allowSnapshot(path);
hdfs.setQuota(path, 3, HdfsConstants.QUOTA_DONT_SET);
FileSystem.create(hdfs, filePath,
FsPermission.createImmutable((short) 0600)).close();
hdfs.createSnapshot(path, snapshotName);
// This adds an XAttr feature, which can throw an exception
exception.expect(NSQuotaExceededException.class);
hdfs.setXAttr(filePath, name1, value1);
}
/**
* Assert exception of removing xattr when exceeding quota.
*/
@Test
public void testRemoveXAttrExceedsQuota() throws Exception {
Path filePath = new Path(path, "file1");
Path fileSnapshotPath = new Path(snapshotPath, "file1");
FileSystem.mkdirs(hdfs, path, FsPermission.createImmutable((short) 0755));
hdfs.allowSnapshot(path);
hdfs.setQuota(path, 3, HdfsConstants.QUOTA_DONT_SET);
FileSystem.create(hdfs, filePath,
FsPermission.createImmutable((short) 0600)).close();
hdfs.setXAttr(filePath, name1, value1);
hdfs.createSnapshot(path, snapshotName);
byte[] value = hdfs.getXAttr(filePath, name1);
Assert.assertArrayEquals(value, value1);
value = hdfs.getXAttr(fileSnapshotPath, name1);
Assert.assertArrayEquals(value, value1);
exception.expect(NSQuotaExceededException.class);
hdfs.removeXAttr(filePath, name1);
}
/** /**
* Test that users can copy a snapshot while preserving its xattrs. * Test that users can copy a snapshot while preserving its xattrs.
*/ */