Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-8357. Consolidate parameters of INode.CleanSubtree() into a parameter objects. Contributed by Li Lu.

This commit is contained in:
Haohui Mai 2015-05-09 22:51:58 -07:00
parent 3cf4f24a9f
commit 1f499a61dc
18 changed files with 231 additions and 290 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -207,6 +207,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8327. Simplify quota calculations for snapshots and truncate. (wheat9) HDFS-8327. Simplify quota calculations for snapshots and truncate. (wheat9)
HDFS-8357. Consolidate parameters of INode.CleanSubtree() into a parameter
objects. (Li Lu via wheat9)
OPTIMIZATIONS OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than

11
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirDeleteOp.java
View File

@ -250,13 +250,14 @@ private static long unprotectedDelete(
} }
// collect block and update quota // collect block and update quota
INode.ReclaimContext reclaimContext = new INode.ReclaimContext(
fsd.getBlockStoragePolicySuite(), collectedBlocks,
removedINodes, removedUCFiles);
if (!targetNode.isInLatestSnapshot(latestSnapshot)) { if (!targetNode.isInLatestSnapshot(latestSnapshot)) {
targetNode.destroyAndCollectBlocks(fsd.getBlockStoragePolicySuite(), targetNode.destroyAndCollectBlocks(reclaimContext);
collectedBlocks, removedINodes, removedUCFiles);
} else { } else {
QuotaCounts counts = targetNode.cleanSubtree( QuotaCounts counts = targetNode.cleanSubtree(reclaimContext,
fsd.getBlockStoragePolicySuite(), CURRENT_STATE_ID, CURRENT_STATE_ID, latestSnapshot);
latestSnapshot, collectedBlocks, removedINodes, removedUCFiles);
removed = counts.getNameSpace(); removed = counts.getNameSpace();
fsd.updateCountNoQuotaCheck(iip, iip.length() -1, counts.negation()); fsd.updateCountNoQuotaCheck(iip, iip.length() -1, counts.negation());
} }

12
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
View File

@ -732,14 +732,16 @@ boolean cleanDst(BlockStoragePolicySuite bsps, BlocksMapUpdateInfo collectedBloc
List<Long> removedUCFiles = new ChunkedArrayList<>(); List<Long> removedUCFiles = new ChunkedArrayList<>();
final boolean filesDeleted; final boolean filesDeleted;
if (!oldDstChild.isInLatestSnapshot(dstIIP.getLatestSnapshotId())) { if (!oldDstChild.isInLatestSnapshot(dstIIP.getLatestSnapshotId())) {
oldDstChild.destroyAndCollectBlocks(bsps, collectedBlocks, removedINodes, oldDstChild.destroyAndCollectBlocks(
removedUCFiles); new INode.ReclaimContext(bsps, collectedBlocks, removedINodes, removedUCFiles));
filesDeleted = true; filesDeleted = true;
} else { } else {
filesDeleted = oldDstChild.cleanSubtree( filesDeleted = oldDstChild.cleanSubtree(
bsps, Snapshot.CURRENT_STATE_ID, new INode.ReclaimContext(bsps, collectedBlocks, removedINodes,
dstIIP.getLatestSnapshotId(), collectedBlocks, removedUCFiles),
removedINodes, removedUCFiles).getNameSpace() >= 0; Snapshot.CURRENT_STATE_ID,
dstIIP.getLatestSnapshotId())
.getNameSpace() >= 0;
} }
fsd.getFSNamesystem().removeLeasesAndINodes( fsd.getFSNamesystem().removeLeasesAndINodes(
removedUCFiles, removedINodes, false); removedUCFiles, removedINodes, false);

85
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
View File

@ -387,30 +387,20 @@ public INodeSymlink asSymlink() {
* snapshot in its diff list. Recursively clean its children. * snapshot in its diff list. Recursively clean its children.
* </pre> * </pre>
* *
* @param bsps * @param reclaimContext
* block storage policy suite to calculate intended storage type usage * Record blocks and inodes that need to be reclaimed.
* @param snapshotId * @param snapshotId
* The id of the snapshot to delete. * The id of the snapshot to delete.
* {@link Snapshot#CURRENT_STATE_ID} means to delete the current * {@link Snapshot#CURRENT_STATE_ID} means to delete the current
* file/directory. * file/directory.
* @param priorSnapshotId * @param priorSnapshotId
* The id of the latest snapshot before the to-be-deleted snapshot. * The id of the latest snapshot before the to-be-deleted snapshot.
* When deleting a current inode, this parameter captures the latest * When deleting a current inode, this parameter captures the latest
* snapshot. * snapshot.
* @param collectedBlocks
* blocks collected from the descents for further block
* deletion/update will be added to the given map.
* @param removedINodes
* INodes collected from the descents for further cleaning up of
* inodeMap
* @param removedUCFiles
* INodes whose leases need to be released
* @return quota usage delta when deleting a snapshot * @return quota usage delta when deleting a snapshot
*/ */
public abstract QuotaCounts cleanSubtree( public abstract QuotaCounts cleanSubtree(
final BlockStoragePolicySuite bsps, final int snapshotId, ReclaimContext reclaimContext, final int snapshotId, int priorSnapshotId);
int priorSnapshotId, BlocksMapUpdateInfo collectedBlocks,
List<INode> removedINodes, List<Long> removedUCFiles);
/** /**
* Destroy self and clear everything! If the INode is a file, this method * Destroy self and clear everything! If the INode is a file, this method
@ -418,22 +408,11 @@ public abstract QuotaCounts cleanSubtree(
* directory, the method goes down the subtree and collects blocks from the * directory, the method goes down the subtree and collects blocks from the
* descents, and clears its parent/children references as well. The method * descents, and clears its parent/children references as well. The method
* also clears the diff list if the INode contains snapshot diff list. * also clears the diff list if the INode contains snapshot diff list.
* @param bsps *
* block storage policy suite to calculate intended storage type usage * @param reclaimContext
* This is needed because INodeReference#destroyAndCollectBlocks() needs * Record blocks and inodes that need to be reclaimed.
* to call INode#cleanSubtree(), which calls INode#computeQuotaUsage().
* @param collectedBlocks
* blocks collected from the descents for further block
* deletion/update will be added to this map.
* @param removedINodes
* INodes collected from the descents for further cleaning up of
* inodeMap
* @param removedUCFiles
* INodes whose leases need to be released
*/ */
public abstract void destroyAndCollectBlocks( public abstract void destroyAndCollectBlocks(ReclaimContext reclaimContext);
BlockStoragePolicySuite bsps, BlocksMapUpdateInfo collectedBlocks,
List<INode> removedINodes, List<Long> removedUCFiles);
/** Compute {@link ContentSummary}. Blocking call */ /** Compute {@link ContentSummary}. Blocking call */
public final ContentSummary computeContentSummary(BlockStoragePolicySuite bsps) { public final ContentSummary computeContentSummary(BlockStoragePolicySuite bsps) {
@ -824,6 +803,44 @@ public void dumpTreeRecursively(PrintWriter out, StringBuilder prefix,
out.print(", " + getPermissionStatus(snapshotId)); out.print(", " + getPermissionStatus(snapshotId));
} }
/**
* Context object to record blocks and inodes that need to be reclaimed
*/
public static class ReclaimContext {
protected final BlockStoragePolicySuite bsps;
protected final BlocksMapUpdateInfo collectedBlocks;
protected final List<INode> removedINodes;
protected final List<Long> removedUCFiles;
/**
* @param bsps
* block storage policy suite to calculate intended storage type
* usage
* @param collectedBlocks
* blocks collected from the descents for further block
* deletion/update will be added to the given map.
* @param removedINodes
* INodes collected from the descents for further cleaning up of
* @param removedUCFiles
* files that the NN need to remove the leases
*/
public ReclaimContext(
BlockStoragePolicySuite bsps, BlocksMapUpdateInfo collectedBlocks,
List<INode> removedINodes, List<Long> removedUCFiles) {
this.bsps = bsps;
this.collectedBlocks = collectedBlocks;
this.removedINodes = removedINodes;
this.removedUCFiles = removedUCFiles;
}
public BlockStoragePolicySuite storagePolicySuite() {
return bsps;
}
public BlocksMapUpdateInfo collectedBlocks() {
return collectedBlocks;
}
}
/** /**
* Information used for updating the blocksMap when deleting files. * Information used for updating the blocksMap when deleting files.
*/ */

44
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
View File

@ -264,11 +264,11 @@ public Snapshot addSnapshot(int id, String name) throws SnapshotException,
return getDirectorySnapshottableFeature().addSnapshot(this, id, name); return getDirectorySnapshottableFeature().addSnapshot(this, id, name);
} }
public Snapshot removeSnapshot(BlockStoragePolicySuite bsps, String snapshotName, public Snapshot removeSnapshot(
BlocksMapUpdateInfo collectedBlocks, final List<INode> removedINodes) ReclaimContext reclaimContext, String snapshotName)
throws SnapshotException { throws SnapshotException {
return getDirectorySnapshottableFeature().removeSnapshot(bsps, this, return getDirectorySnapshottableFeature().removeSnapshot(
snapshotName, collectedBlocks, removedINodes); reclaimContext, this, snapshotName);
} }
public void renameSnapshot(String path, String oldName, String newName) public void renameSnapshot(String path, String oldName, String newName)
@ -754,9 +754,7 @@ public void clear() {
/** Call cleanSubtree(..) recursively down the subtree. */ /** Call cleanSubtree(..) recursively down the subtree. */
public QuotaCounts cleanSubtreeRecursively( public QuotaCounts cleanSubtreeRecursively(
final BlockStoragePolicySuite bsps, final int snapshot, int prior, ReclaimContext reclaimContext, final int snapshot, int prior,
final BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes, List<Long> removedUCFiles,
final Map<INode, INode> excludedNodes) { final Map<INode, INode> excludedNodes) {
QuotaCounts counts = new QuotaCounts.Builder().build(); QuotaCounts counts = new QuotaCounts.Builder().build();
// in case of deletion snapshot, since this call happens after we modify // in case of deletion snapshot, since this call happens after we modify
@ -771,8 +769,7 @@ public QuotaCounts cleanSubtreeRecursively(
&& excludedNodes.containsKey(child)) { && excludedNodes.containsKey(child)) {
continue; continue;
} else { } else {
QuotaCounts childCounts = child.cleanSubtree(bsps, snapshot, prior, QuotaCounts childCounts = child.cleanSubtree(reclaimContext, snapshot, prior);
collectedBlocks, removedINodes, removedUCFiles);
counts.add(childCounts); counts.add(childCounts);
} }
} }
@ -780,49 +777,42 @@ public QuotaCounts cleanSubtreeRecursively(
} }
@Override @Override
public void destroyAndCollectBlocks( public void destroyAndCollectBlocks(ReclaimContext reclaimContext) {
final BlockStoragePolicySuite bsps,
final BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes, List<Long> removedUCFiles) {
final DirectoryWithSnapshotFeature sf = getDirectoryWithSnapshotFeature(); final DirectoryWithSnapshotFeature sf = getDirectoryWithSnapshotFeature();
if (sf != null) { if (sf != null) {
sf.clear(bsps, this, collectedBlocks, removedINodes, removedUCFiles); sf.clear(reclaimContext, this);
} }
for (INode child : getChildrenList(Snapshot.CURRENT_STATE_ID)) { for (INode child : getChildrenList(Snapshot.CURRENT_STATE_ID)) {
child.destroyAndCollectBlocks(bsps, collectedBlocks, removedINodes, child.destroyAndCollectBlocks(reclaimContext);
removedUCFiles);
} }
if (getAclFeature() != null) { if (getAclFeature() != null) {
AclStorage.removeAclFeature(getAclFeature()); AclStorage.removeAclFeature(getAclFeature());
} }
clear(); clear();
removedINodes.add(this); reclaimContext.removedINodes.add(this);
} }
@Override @Override
public QuotaCounts cleanSubtree( public QuotaCounts cleanSubtree(
final BlockStoragePolicySuite bsps, final int snapshotId, int priorSnapshotId, ReclaimContext reclaimContext, final int snapshotId, int priorSnapshotId) {
final BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes, List<Long> removedUCFiles) {
DirectoryWithSnapshotFeature sf = getDirectoryWithSnapshotFeature(); DirectoryWithSnapshotFeature sf = getDirectoryWithSnapshotFeature();
// there is snapshot data // there is snapshot data
if (sf != null) { if (sf != null) {
return sf.cleanDirectory(bsps, this, snapshotId, priorSnapshotId, return sf.cleanDirectory(reclaimContext, this, snapshotId,
collectedBlocks, removedINodes, removedUCFiles); priorSnapshotId);
} }
// there is no snapshot data // there is no snapshot data
if (priorSnapshotId == Snapshot.NO_SNAPSHOT_ID if (priorSnapshotId == Snapshot.NO_SNAPSHOT_ID
&& snapshotId == Snapshot.CURRENT_STATE_ID) { && snapshotId == Snapshot.CURRENT_STATE_ID) {
// destroy the whole subtree and collect blocks that should be deleted // destroy the whole subtree and collect blocks that should be deleted
QuotaCounts counts = new QuotaCounts.Builder().build(); QuotaCounts counts = new QuotaCounts.Builder().build();
this.computeQuotaUsage(bsps, counts, true); this.computeQuotaUsage(reclaimContext.bsps, counts, true);
destroyAndCollectBlocks(bsps, collectedBlocks, removedINodes, destroyAndCollectBlocks(reclaimContext);
removedUCFiles);
return counts; return counts;
} else { } else {
// process recursively down the subtree // process recursively down the subtree
QuotaCounts counts = cleanSubtreeRecursively(bsps, snapshotId, priorSnapshotId, QuotaCounts counts = cleanSubtreeRecursively(
collectedBlocks, removedINodes, removedUCFiles, null); reclaimContext, snapshotId, priorSnapshotId, null);
if (isQuotaSet()) { if (isQuotaSet()) {
getDirectoryWithQuotaFeature().addSpaceConsumed2Cache(counts.negation()); getDirectoryWithQuotaFeature().addSpaceConsumed2Cache(counts.negation());
} }

35
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
View File

@ -492,13 +492,10 @@ public void setBlocks(BlockInfoContiguous[] blocks) {
@Override @Override
public QuotaCounts cleanSubtree( public QuotaCounts cleanSubtree(
BlockStoragePolicySuite bsps, final int snapshot, int priorSnapshotId, ReclaimContext reclaimContext, final int snapshot, int priorSnapshotId) {
final BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes, List<Long> removedUCFiles) {
FileWithSnapshotFeature sf = getFileWithSnapshotFeature(); FileWithSnapshotFeature sf = getFileWithSnapshotFeature();
if (sf != null) { if (sf != null) {
return sf.cleanFile(bsps, this, snapshot, priorSnapshotId, collectedBlocks, return sf.cleanFile(reclaimContext, this, snapshot, priorSnapshotId);
removedINodes);
} }
QuotaCounts counts = new QuotaCounts.Builder().build(); QuotaCounts counts = new QuotaCounts.Builder().build();
@ -506,17 +503,16 @@ public QuotaCounts cleanSubtree(
if (priorSnapshotId == NO_SNAPSHOT_ID) { if (priorSnapshotId == NO_SNAPSHOT_ID) {
// this only happens when deleting the current file and the file is not // this only happens when deleting the current file and the file is not
// in any snapshot // in any snapshot
computeQuotaUsage(bsps, counts, false); computeQuotaUsage(reclaimContext.bsps, counts, false);
destroyAndCollectBlocks(bsps, collectedBlocks, removedINodes, destroyAndCollectBlocks(reclaimContext);
removedUCFiles);
} else { } else {
FileUnderConstructionFeature uc = getFileUnderConstructionFeature(); FileUnderConstructionFeature uc = getFileUnderConstructionFeature();
// when deleting the current file and the file is in snapshot, we should // when deleting the current file and the file is in snapshot, we should
// clean the 0-sized block if the file is UC // clean the 0-sized block if the file is UC
if (uc != null) { if (uc != null) {
uc.cleanZeroSizeBlock(this, collectedBlocks); uc.cleanZeroSizeBlock(this, reclaimContext.collectedBlocks);
if (removedUCFiles != null) { if (reclaimContext.removedUCFiles != null) {
removedUCFiles.add(getId()); reclaimContext.removedUCFiles.add(getId());
} }
} }
} }
@ -525,12 +521,10 @@ public QuotaCounts cleanSubtree(
} }
@Override @Override
public void destroyAndCollectBlocks( public void destroyAndCollectBlocks(ReclaimContext reclaimContext) {
BlockStoragePolicySuite bsps, BlocksMapUpdateInfo collectedBlocks, if (blocks != null && reclaimContext.collectedBlocks != null) {
final List<INode> removedINodes, List<Long> removedUCFiles) {
if (blocks != null && collectedBlocks != null) {
for (BlockInfoContiguous blk : blocks) { for (BlockInfoContiguous blk : blocks) {
collectedBlocks.addDeleteBlock(blk); reclaimContext.collectedBlocks.addDeleteBlock(blk);
blk.setBlockCollection(null); blk.setBlockCollection(null);
} }
} }
@ -539,14 +533,15 @@ public void destroyAndCollectBlocks(
AclStorage.removeAclFeature(getAclFeature()); AclStorage.removeAclFeature(getAclFeature());
} }
clear(); clear();
removedINodes.add(this); reclaimContext.removedINodes.add(this);
FileWithSnapshotFeature sf = getFileWithSnapshotFeature(); FileWithSnapshotFeature sf = getFileWithSnapshotFeature();
if (sf != null) { if (sf != null) {
sf.getDiffs().destroyAndCollectSnapshotBlocks(collectedBlocks); sf.getDiffs().destroyAndCollectSnapshotBlocks(
reclaimContext.collectedBlocks);
sf.clearDiffs(); sf.clearDiffs();
} }
if (isUnderConstruction() && removedUCFiles != null) { if (isUnderConstruction() && reclaimContext.removedUCFiles != null) {
removedUCFiles.add(getId()); reclaimContext.removedUCFiles.add(getId());
} }
} }

9
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java
View File

@ -18,7 +18,6 @@
package org.apache.hadoop.hdfs.server.namenode; package org.apache.hadoop.hdfs.server.namenode;
import java.util.Iterator; import java.util.Iterator;
import java.util.List;
import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.fs.permission.PermissionStatus;
@ -97,9 +96,7 @@ void recordModification(int latestSnapshotId) {
} }
@Override @Override
public void destroyAndCollectBlocks( public void destroyAndCollectBlocks(ReclaimContext reclaimContext) {
BlockStoragePolicySuite bsps, BlocksMapUpdateInfo collectedBlocks,
List<INode> removedINodes, List<Long> removedUCFiles) {
// Nothing to do // Nothing to do
} }
@ -118,9 +115,7 @@ public ContentSummaryComputationContext computeContentSummary(
@Override @Override
public QuotaCounts cleanSubtree( public QuotaCounts cleanSubtree(
BlockStoragePolicySuite bsps, int snapshotId, int priorSnapshotId, ReclaimContext reclaimContext, int snapshotId, int priorSnapshotId) {
BlocksMapUpdateInfo collectedBlocks, List<INode> removedINodes,
List<Long> removedUCFiles) {
return null; return null;
} }

59
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java
View File

@ -303,19 +303,15 @@ final void recordModification(int latestSnapshotId) {
@Override // used by WithCount @Override // used by WithCount
public QuotaCounts cleanSubtree( public QuotaCounts cleanSubtree(
BlockStoragePolicySuite bsps, int snapshot, int prior, BlocksMapUpdateInfo collectedBlocks, ReclaimContext reclaimContext, int snapshot, int prior) {
final List<INode> removedINodes, List<Long> removedUCFiles) { return referred.cleanSubtree(reclaimContext,
return referred.cleanSubtree(bsps, snapshot, prior, collectedBlocks, snapshot, prior);
removedINodes, removedUCFiles);
} }
@Override // used by WithCount @Override // used by WithCount
public void destroyAndCollectBlocks( public void destroyAndCollectBlocks(ReclaimContext reclaimContext) {
BlockStoragePolicySuite bsps, BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes, List<Long> removedUCFiles) {
if (removeReference(this) <= 0) { if (removeReference(this) <= 0) {
referred.destroyAndCollectBlocks(bsps, collectedBlocks, removedINodes, referred.destroyAndCollectBlocks(reclaimContext);
removedUCFiles);
} }
} }
@ -544,8 +540,7 @@ public final QuotaCounts computeQuotaUsage(BlockStoragePolicySuite bsps,
@Override @Override
public QuotaCounts cleanSubtree( public QuotaCounts cleanSubtree(
BlockStoragePolicySuite bsps, final int snapshot, int prior, final BlocksMapUpdateInfo collectedBlocks, ReclaimContext reclaimContext, final int snapshot, int prior) {
final List<INode> removedINodes, List<Long> removedUCFiles) {
// since WithName node resides in deleted list acting as a snapshot copy, // since WithName node resides in deleted list acting as a snapshot copy,
// the parameter snapshot must be non-null // the parameter snapshot must be non-null
Preconditions.checkArgument(snapshot != Snapshot.CURRENT_STATE_ID); Preconditions.checkArgument(snapshot != Snapshot.CURRENT_STATE_ID);
@ -560,8 +555,8 @@ public QuotaCounts cleanSubtree(
return new QuotaCounts.Builder().build(); return new QuotaCounts.Builder().build();
} }
QuotaCounts counts = getReferredINode().cleanSubtree(bsps, snapshot, prior, QuotaCounts counts = getReferredINode().cleanSubtree(reclaimContext,
collectedBlocks, removedINodes, removedUCFiles); snapshot, prior);
INodeReference ref = getReferredINode().getParentReference(); INodeReference ref = getReferredINode().getParentReference();
if (ref != null) { if (ref != null) {
try { try {
@ -582,13 +577,10 @@ public QuotaCounts cleanSubtree(
} }
@Override @Override
public void destroyAndCollectBlocks( public void destroyAndCollectBlocks(ReclaimContext reclaimContext) {
BlockStoragePolicySuite bsps, BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes, List<Long> removedUCFiles) {
int snapshot = getSelfSnapshot(); int snapshot = getSelfSnapshot();
if (removeReference(this) <= 0) { if (removeReference(this) <= 0) {
getReferredINode().destroyAndCollectBlocks(bsps, collectedBlocks, getReferredINode().destroyAndCollectBlocks(reclaimContext);
removedINodes, removedUCFiles);
} else { } else {
int prior = getPriorSnapshot(this); int prior = getPriorSnapshot(this);
INode referred = getReferredINode().asReference().getReferredINode(); INode referred = getReferredINode().asReference().getReferredINode();
@ -607,8 +599,8 @@ public void destroyAndCollectBlocks(
return; return;
} }
try { try {
QuotaCounts counts = referred.cleanSubtree(bsps, snapshot, prior, QuotaCounts counts = referred.cleanSubtree(reclaimContext,
collectedBlocks, removedINodes, removedUCFiles); snapshot, prior);
INodeReference ref = getReferredINode().getParentReference(); INodeReference ref = getReferredINode().getParentReference();
if (ref != null) { if (ref != null) {
ref.addSpaceConsumed(counts.negation(), true); ref.addSpaceConsumed(counts.negation(), true);
@ -663,15 +655,12 @@ public DstReference(INodeDirectory parent, WithCount referred,
@Override @Override
public QuotaCounts cleanSubtree( public QuotaCounts cleanSubtree(
BlockStoragePolicySuite bsps, int snapshot, int prior, ReclaimContext reclaimContext, int snapshot, int prior) {
BlocksMapUpdateInfo collectedBlocks, List<INode> removedINodes,
List<Long> removedUCFiles) {
if (snapshot == Snapshot.CURRENT_STATE_ID if (snapshot == Snapshot.CURRENT_STATE_ID
&& prior == Snapshot.NO_SNAPSHOT_ID) { && prior == Snapshot.NO_SNAPSHOT_ID) {
QuotaCounts counts = new QuotaCounts.Builder().build(); QuotaCounts counts = new QuotaCounts.Builder().build();
this.computeQuotaUsage(bsps, counts, true); this.computeQuotaUsage(reclaimContext.bsps, counts, true);
destroyAndCollectBlocks(bsps, collectedBlocks, removedINodes, destroyAndCollectBlocks(reclaimContext);
removedUCFiles);
return counts; return counts;
} else { } else {
// if prior is NO_SNAPSHOT_ID, we need to check snapshot belonging to // if prior is NO_SNAPSHOT_ID, we need to check snapshot belonging to
@ -687,8 +676,7 @@ public QuotaCounts cleanSubtree(
&& Snapshot.ID_INTEGER_COMPARATOR.compare(snapshot, prior) <= 0) { && Snapshot.ID_INTEGER_COMPARATOR.compare(snapshot, prior) <= 0) {
return new QuotaCounts.Builder().build(); return new QuotaCounts.Builder().build();
} }
return getReferredINode().cleanSubtree(bsps, snapshot, prior, return getReferredINode().cleanSubtree(reclaimContext, snapshot, prior);
collectedBlocks, removedINodes, removedUCFiles);
} }
} }
@ -701,14 +689,12 @@ public QuotaCounts cleanSubtree(
* referred node's subtree and delete everything created after the last * referred node's subtree and delete everything created after the last
* rename operation, i.e., everything outside of the scope of the prior * rename operation, i.e., everything outside of the scope of the prior
* WithName nodes. * WithName nodes.
* @param reclaimContext
*/ */
@Override @Override
public void destroyAndCollectBlocks( public void destroyAndCollectBlocks(ReclaimContext reclaimContext) {
BlockStoragePolicySuite bsps, BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes, List<Long> removedUCFiles) {
if (removeReference(this) <= 0) { if (removeReference(this) <= 0) {
getReferredINode().destroyAndCollectBlocks(bsps, collectedBlocks, getReferredINode().destroyAndCollectBlocks(reclaimContext);
removedINodes, removedUCFiles);
} else { } else {
// we will clean everything, including files, directories, and // we will clean everything, including files, directories, and
// snapshots, that were created after this prior snapshot // snapshots, that were created after this prior snapshot
@ -730,16 +716,15 @@ public void destroyAndCollectBlocks(
// when calling cleanSubtree of the referred node, since we // when calling cleanSubtree of the referred node, since we
// compute quota usage updates before calling this destroy // compute quota usage updates before calling this destroy
// function, we use true for countDiffChange // function, we use true for countDiffChange
referred.cleanSubtree(bsps, snapshot, prior, collectedBlocks, referred.cleanSubtree(reclaimContext, snapshot, prior);
removedINodes, removedUCFiles);
} else if (referred.isDirectory()) { } else if (referred.isDirectory()) {
// similarly, if referred is a directory, it must be an // similarly, if referred is a directory, it must be an
// INodeDirectory with snapshot // INodeDirectory with snapshot
INodeDirectory dir = referred.asDirectory(); INodeDirectory dir = referred.asDirectory();
Preconditions.checkState(dir.isWithSnapshot()); Preconditions.checkState(dir.isWithSnapshot());
try { try {
DirectoryWithSnapshotFeature.destroyDstSubtree(bsps, dir, snapshot, DirectoryWithSnapshotFeature.destroyDstSubtree(
prior, collectedBlocks, removedINodes, removedUCFiles); reclaimContext, dir, snapshot, prior);
} catch (QuotaExceededException e) { } catch (QuotaExceededException e) {
LOG.error("should not exceed quota while snapshot deletion", e); LOG.error("should not exceed quota while snapshot deletion", e);
} }

15
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java
View File

@ -18,7 +18,6 @@
package org.apache.hadoop.hdfs.server.namenode; package org.apache.hadoop.hdfs.server.namenode;
import java.io.PrintWriter; import java.io.PrintWriter;
import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.fs.permission.PermissionStatus;
@ -74,23 +73,17 @@ public byte[] getSymlink() {
@Override @Override
public QuotaCounts cleanSubtree( public QuotaCounts cleanSubtree(
BlockStoragePolicySuite bsps, final int snapshotId, int priorSnapshotId, ReclaimContext reclaimContext, final int snapshotId, int priorSnapshotId) {
final BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes, List<Long> removedUCFiles) {
if (snapshotId == Snapshot.CURRENT_STATE_ID if (snapshotId == Snapshot.CURRENT_STATE_ID
&& priorSnapshotId == Snapshot.NO_SNAPSHOT_ID) { && priorSnapshotId == Snapshot.NO_SNAPSHOT_ID) {
destroyAndCollectBlocks(bsps, collectedBlocks, removedINodes, destroyAndCollectBlocks(reclaimContext);
removedUCFiles);
} }
return new QuotaCounts.Builder().nameSpace(1).build(); return new QuotaCounts.Builder().nameSpace(1).build();
} }
@Override @Override
public void destroyAndCollectBlocks( public void destroyAndCollectBlocks(ReclaimContext reclaimContext) {
final BlockStoragePolicySuite bsps, reclaimContext.removedINodes.add(this);
final BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes, List<Long> removedUCFiles) {
removedINodes.add(this);
} }
@Override @Override

15
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiff.java
View File

@ -116,21 +116,18 @@ A getSnapshotINode() {
/** Combine the posterior diff and collect blocks for deletion. */ /** Combine the posterior diff and collect blocks for deletion. */
abstract QuotaCounts combinePosteriorAndCollectBlocks( abstract QuotaCounts combinePosteriorAndCollectBlocks(
final BlockStoragePolicySuite bsps, final N currentINode, INode.ReclaimContext reclaimContext, final N currentINode,
final D posterior, final BlocksMapUpdateInfo collectedBlocks, final D posterior);
final List<INode> removedINodes);
/** /**
* Delete and clear self. * Delete and clear self.
* @param bsps The block storage policy suite used to retrieve storage policy * @param reclaimContext blocks and inodes that need to be reclaimed
* @param currentINode The inode where the deletion happens. * @param currentINode The inode where the deletion happens.
* @param collectedBlocks Used to collect blocks for deletion. *
* @param removedINodes INodes removed * @return usage delta
* @return quota usage delta
*/ */
abstract QuotaCounts destroyDiffAndCollectBlocks( abstract QuotaCounts destroyDiffAndCollectBlocks(
final BlockStoragePolicySuite bsps, final N currentINode, INode.ReclaimContext reclaimContext, final N currentINode);
final BlocksMapUpdateInfo collectedBlocks, final List<INode> removedINodes);
@Override @Override
public String toString() { public String toString() {

20
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java
View File

@ -22,9 +22,7 @@
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.server.namenode.INode; import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
import org.apache.hadoop.hdfs.server.namenode.INodeAttributes; import org.apache.hadoop.hdfs.server.namenode.INodeAttributes;
import org.apache.hadoop.hdfs.server.namenode.QuotaCounts; import org.apache.hadoop.hdfs.server.namenode.QuotaCounts;
@ -62,16 +60,14 @@ public void clear() {
* outside. If the diff to remove is not the first one in the diff list, we * outside. If the diff to remove is not the first one in the diff list, we
* need to combine the diff with its previous one. * need to combine the diff with its previous one.
* *
* @param reclaimContext blocks and inodes that need to be reclaimed
* @param snapshot The id of the snapshot to be deleted * @param snapshot The id of the snapshot to be deleted
* @param prior The id of the snapshot taken before the to-be-deleted snapshot * @param prior The id of the snapshot taken before the to-be-deleted snapshot
* @param collectedBlocks Used to collect information for blocksMap update
* @return delta in namespace. * @return delta in namespace.
*/ */
public final QuotaCounts deleteSnapshotDiff(BlockStoragePolicySuite bsps, public final QuotaCounts deleteSnapshotDiff(
final int snapshot, INode.ReclaimContext reclaimContext, final int snapshot, final int prior,
final int prior, final N currentINode, final N currentINode) {
final BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes) {
int snapshotIndex = Collections.binarySearch(diffs, snapshot); int snapshotIndex = Collections.binarySearch(diffs, snapshot);
QuotaCounts counts = new QuotaCounts.Builder().build(); QuotaCounts counts = new QuotaCounts.Builder().build();
@ -82,8 +78,8 @@ public final QuotaCounts deleteSnapshotDiff(BlockStoragePolicySuite bsps,
diffs.get(snapshotIndex).setSnapshotId(prior); diffs.get(snapshotIndex).setSnapshotId(prior);
} else { // there is no snapshot before } else { // there is no snapshot before
removed = diffs.remove(0); removed = diffs.remove(0);
counts.add(removed.destroyDiffAndCollectBlocks(bsps, currentINode, counts.add(removed.destroyDiffAndCollectBlocks(reclaimContext,
collectedBlocks, removedINodes)); currentINode));
} }
} else if (snapshotIndex > 0) { } else if (snapshotIndex > 0) {
final AbstractINodeDiff<N, A, D> previous = diffs.get(snapshotIndex - 1); final AbstractINodeDiff<N, A, D> previous = diffs.get(snapshotIndex - 1);
@ -96,8 +92,8 @@ public final QuotaCounts deleteSnapshotDiff(BlockStoragePolicySuite bsps,
previous.snapshotINode = removed.snapshotINode; previous.snapshotINode = removed.snapshotINode;
} }
counts.add(previous.combinePosteriorAndCollectBlocks( counts.add(previous.combinePosteriorAndCollectBlocks(reclaimContext,
bsps, currentINode, removed, collectedBlocks, removedINodes)); currentINode, removed));
previous.setPosterior(removed.getPosterior()); previous.setPosterior(removed.getPosterior());
removed.setPosterior(null); removed.setPosterior(null);
} }

12
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java
View File

@ -197,15 +197,15 @@ public Snapshot addSnapshot(INodeDirectory snapshotRoot, int id, String name)
* Remove the snapshot with the given name from {@link #snapshotsByNames}, * Remove the snapshot with the given name from {@link #snapshotsByNames},
* and delete all the corresponding DirectoryDiff. * and delete all the corresponding DirectoryDiff.
* *
* @param reclaimContext records blocks and inodes that need to be reclaimed
* @param snapshotRoot The directory where we take snapshots * @param snapshotRoot The directory where we take snapshots
* @param snapshotName The name of the snapshot to be removed * @param snapshotName The name of the snapshot to be removed
* @param collectedBlocks Used to collect information to update blocksMap
* @return The removed snapshot. Null if no snapshot with the given name * @return The removed snapshot. Null if no snapshot with the given name
* exists. * exists.
*/ */
public Snapshot removeSnapshot(BlockStoragePolicySuite bsps, INodeDirectory snapshotRoot, public Snapshot removeSnapshot(
String snapshotName, BlocksMapUpdateInfo collectedBlocks, INode.ReclaimContext reclaimContext, INodeDirectory snapshotRoot,
final List<INode> removedINodes) throws SnapshotException { String snapshotName) throws SnapshotException {
final int i = searchSnapshot(DFSUtil.string2Bytes(snapshotName)); final int i = searchSnapshot(DFSUtil.string2Bytes(snapshotName));
if (i < 0) { if (i < 0) {
throw new SnapshotException("Cannot delete snapshot " + snapshotName throw new SnapshotException("Cannot delete snapshot " + snapshotName
@ -215,8 +215,8 @@ public Snapshot removeSnapshot(BlockStoragePolicySuite bsps, INodeDirectory snap
final Snapshot snapshot = snapshotsByNames.get(i); final Snapshot snapshot = snapshotsByNames.get(i);
int prior = Snapshot.findLatestSnapshot(snapshotRoot, snapshot.getId()); int prior = Snapshot.findLatestSnapshot(snapshotRoot, snapshot.getId());
try { try {
QuotaCounts counts = snapshotRoot.cleanSubtree(bsps, snapshot.getId(), QuotaCounts counts = snapshotRoot.cleanSubtree(reclaimContext,
prior, collectedBlocks, removedINodes, null); snapshot.getId(), prior);
INodeDirectory parent = snapshotRoot.getParent(); INodeDirectory parent = snapshotRoot.getParent();
if (parent != null) { if (parent != null) {
// there will not be any WithName node corresponding to the deleted // there will not be any WithName node corresponding to the deleted

121
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
View File

@ -35,7 +35,6 @@
import org.apache.hadoop.hdfs.server.namenode.ContentSummaryComputationContext; import org.apache.hadoop.hdfs.server.namenode.ContentSummaryComputationContext;
import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization; import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization;
import org.apache.hadoop.hdfs.server.namenode.INode; import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
import org.apache.hadoop.hdfs.server.namenode.INodeDirectoryAttributes; import org.apache.hadoop.hdfs.server.namenode.INodeDirectoryAttributes;
import org.apache.hadoop.hdfs.server.namenode.INodeFile; import org.apache.hadoop.hdfs.server.namenode.INodeFile;
@ -97,15 +96,12 @@ private boolean removeChild(ListType type, final INode child) {
/** clear the created list */ /** clear the created list */
private QuotaCounts destroyCreatedList( private QuotaCounts destroyCreatedList(
final BlockStoragePolicySuite bsps, final INodeDirectory currentINode, INode.ReclaimContext reclaimContext, final INodeDirectory currentINode) {
final BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes, List<Long> removedUCFiles) {
QuotaCounts counts = new QuotaCounts.Builder().build(); QuotaCounts counts = new QuotaCounts.Builder().build();
final List<INode> createdList = getList(ListType.CREATED); final List<INode> createdList = getList(ListType.CREATED);
for (INode c : createdList) { for (INode c : createdList) {
c.computeQuotaUsage(bsps, counts, true); c.computeQuotaUsage(reclaimContext.storagePolicySuite(), counts, true);
c.destroyAndCollectBlocks(bsps, collectedBlocks, removedINodes, c.destroyAndCollectBlocks(reclaimContext);
removedUCFiles);
// c should be contained in the children list, remove it // c should be contained in the children list, remove it
currentINode.removeChild(c); currentINode.removeChild(c);
} }
@ -114,16 +110,12 @@ private QuotaCounts destroyCreatedList(
} }
/** clear the deleted list */ /** clear the deleted list */
private QuotaCounts destroyDeletedList( private QuotaCounts destroyDeletedList(INode.ReclaimContext reclaimContext) {
final BlockStoragePolicySuite bsps,
final BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes, List<Long> removedUCFiles) {
QuotaCounts counts = new QuotaCounts.Builder().build(); QuotaCounts counts = new QuotaCounts.Builder().build();
final List<INode> deletedList = getList(ListType.DELETED); final List<INode> deletedList = getList(ListType.DELETED);
for (INode d : deletedList) { for (INode d : deletedList) {
d.computeQuotaUsage(bsps, counts, false); d.computeQuotaUsage(reclaimContext.storagePolicySuite(), counts, false);
d.destroyAndCollectBlocks(bsps, collectedBlocks, removedINodes, d.destroyAndCollectBlocks(reclaimContext);
removedUCFiles);
} }
deletedList.clear(); deletedList.clear();
return counts; return counts;
@ -211,19 +203,17 @@ boolean isSnapshotRoot() {
@Override @Override
QuotaCounts combinePosteriorAndCollectBlocks( QuotaCounts combinePosteriorAndCollectBlocks(
final BlockStoragePolicySuite bsps, final INodeDirectory currentDir, final INode.ReclaimContext reclaimContext,
final DirectoryDiff posterior, final INodeDirectory currentDir,
final BlocksMapUpdateInfo collectedBlocks, final DirectoryDiff posterior) {
final List<INode> removedINodes) {
final QuotaCounts counts = new QuotaCounts.Builder().build(); final QuotaCounts counts = new QuotaCounts.Builder().build();
diff.combinePosterior(posterior.diff, new Diff.Processor<INode>() { diff.combinePosterior(posterior.diff, new Diff.Processor<INode>() {
/** Collect blocks for deleted files. */ /** Collect blocks for deleted files. */
@Override @Override
public void process(INode inode) { public void process(INode inode) {
if (inode != null) { if (inode != null) {
inode.computeQuotaUsage(bsps, counts, false); inode.computeQuotaUsage(reclaimContext.storagePolicySuite(), counts, false);
inode.destroyAndCollectBlocks(bsps, collectedBlocks, removedINodes, inode.destroyAndCollectBlocks(reclaimContext);
null);
} }
} }
}); });
@ -322,12 +312,10 @@ void write(DataOutput out, ReferenceMap referenceMap) throws IOException {
@Override @Override
QuotaCounts destroyDiffAndCollectBlocks( QuotaCounts destroyDiffAndCollectBlocks(
BlockStoragePolicySuite bsps, INodeDirectory currentINode, INode.ReclaimContext reclaimContext, INodeDirectory currentINode) {
BlocksMapUpdateInfo collectedBlocks, final List<INode> removedINodes) {
// this diff has been deleted // this diff has been deleted
QuotaCounts counts = new QuotaCounts.Builder().build(); QuotaCounts counts = new QuotaCounts.Builder().build();
counts.add(diff.destroyDeletedList(bsps, collectedBlocks, removedINodes, counts.add(diff.destroyDeletedList(reclaimContext));
null));
INodeDirectoryAttributes snapshotINode = getSnapshotINode(); INodeDirectoryAttributes snapshotINode = getSnapshotINode();
if (snapshotINode != null && snapshotINode.getAclFeature() != null) { if (snapshotINode != null && snapshotINode.getAclFeature() != null) {
AclStorage.removeAclFeature(snapshotINode.getAclFeature()); AclStorage.removeAclFeature(snapshotINode.getAclFeature());
@ -412,25 +400,24 @@ private static Map<INode, INode> cloneDiffList(List<INode> diffList) {
* Destroy a subtree under a DstReference node. * Destroy a subtree under a DstReference node.
*/ */
public static void destroyDstSubtree( public static void destroyDstSubtree(
final BlockStoragePolicySuite bsps, INode inode, final int snapshot, INode.ReclaimContext reclaimContext, INode inode, final int snapshot,
final int prior, final BlocksMapUpdateInfo collectedBlocks, final int prior) throws QuotaExceededException {
final List<INode> removedINodes, List<Long> removedUCFiles) throws QuotaExceededException {
Preconditions.checkArgument(prior != Snapshot.NO_SNAPSHOT_ID); Preconditions.checkArgument(prior != Snapshot.NO_SNAPSHOT_ID);
if (inode.isReference()) { if (inode.isReference()) {
if (inode instanceof INodeReference.WithName if (inode instanceof INodeReference.WithName
&& snapshot != Snapshot.CURRENT_STATE_ID) { && snapshot != Snapshot.CURRENT_STATE_ID) {
// this inode has been renamed before the deletion of the DstReference // this inode has been renamed before the deletion of the DstReference
// subtree // subtree
inode.cleanSubtree(bsps, snapshot, prior, collectedBlocks, removedINodes, inode.cleanSubtree(reclaimContext,
removedUCFiles); snapshot, prior);
} else { } else {
// for DstReference node, continue this process to its subtree // for DstReference node, continue this process to its subtree
destroyDstSubtree(bsps, inode.asReference().getReferredINode(), snapshot, destroyDstSubtree(reclaimContext,
prior, collectedBlocks, removedINodes, removedUCFiles); inode.asReference().getReferredINode(), snapshot,
prior);
} }
} else if (inode.isFile()) { } else if (inode.isFile()) {
inode.cleanSubtree(bsps, snapshot, prior, collectedBlocks, removedINodes, inode.cleanSubtree(reclaimContext, snapshot, prior);
removedUCFiles);
} else if (inode.isDirectory()) { } else if (inode.isDirectory()) {
Map<INode, INode> excludedNodes = null; Map<INode, INode> excludedNodes = null;
INodeDirectory dir = inode.asDirectory(); INodeDirectory dir = inode.asDirectory();
@ -444,21 +431,19 @@ public static void destroyDstSubtree(
} }
if (snapshot != Snapshot.CURRENT_STATE_ID) { if (snapshot != Snapshot.CURRENT_STATE_ID) {
diffList.deleteSnapshotDiff(bsps, snapshot, prior, dir, collectedBlocks, diffList.deleteSnapshotDiff(reclaimContext,
removedINodes); snapshot, prior, dir);
} }
priorDiff = diffList.getDiffById(prior); priorDiff = diffList.getDiffById(prior);
if (priorDiff != null && priorDiff.getSnapshotId() == prior) { if (priorDiff != null && priorDiff.getSnapshotId() == prior) {
priorDiff.diff.destroyCreatedList(bsps, dir, collectedBlocks, priorDiff.diff.destroyCreatedList(reclaimContext, dir);
removedINodes, removedUCFiles);
} }
} }
for (INode child : inode.asDirectory().getChildrenList(prior)) { for (INode child : inode.asDirectory().getChildrenList(prior)) {
if (excludedNodes != null && excludedNodes.containsKey(child)) { if (excludedNodes != null && excludedNodes.containsKey(child)) {
continue; continue;
} }
destroyDstSubtree(bsps, child, snapshot, prior, collectedBlocks, destroyDstSubtree(reclaimContext, child, snapshot, prior);
removedINodes, removedUCFiles);
} }
} }
} }
@ -466,18 +451,15 @@ public static void destroyDstSubtree(
/** /**
* Clean an inode while we move it from the deleted list of post to the * Clean an inode while we move it from the deleted list of post to the
* deleted list of prior. * deleted list of prior.
* @param bsps The block storage policy suite. * @param reclaimContext blocks and inodes that need to be reclaimed
* @param inode The inode to clean. * @param inode The inode to clean.
* @param post The post snapshot. * @param post The post snapshot.
* @param prior The id of the prior snapshot. * @param prior The id of the prior snapshot.
* @param collectedBlocks Used to collect blocks for later deletion.
* @param removedUCFiles
* @return Quota usage update. * @return Quota usage update.
*/ */
private static QuotaCounts cleanDeletedINode( private static QuotaCounts cleanDeletedINode(
final BlockStoragePolicySuite bsps, INode inode, final int post, final int prior, INode.ReclaimContext reclaimContext, INode inode, final int post,
final BlocksMapUpdateInfo collectedBlocks, final int prior) {
final List<INode> removedINodes, List<Long> removedUCFiles) {
QuotaCounts counts = new QuotaCounts.Builder().build(); QuotaCounts counts = new QuotaCounts.Builder().build();
Deque<INode> queue = new ArrayDeque<INode>(); Deque<INode> queue = new ArrayDeque<INode>();
queue.addLast(inode); queue.addLast(inode);
@ -486,15 +468,13 @@ private static QuotaCounts cleanDeletedINode(
if (topNode instanceof INodeReference.WithName) { if (topNode instanceof INodeReference.WithName) {
INodeReference.WithName wn = (INodeReference.WithName) topNode; INodeReference.WithName wn = (INodeReference.WithName) topNode;
if (wn.getLastSnapshotId() >= post) { if (wn.getLastSnapshotId() >= post) {
wn.cleanSubtree(bsps, post, prior, collectedBlocks, removedINodes, wn.cleanSubtree(reclaimContext, post, prior);
removedUCFiles);
} }
// For DstReference node, since the node is not in the created list of // For DstReference node, since the node is not in the created list of
// prior, we should treat it as regular file/dir // prior, we should treat it as regular file/dir
} else if (topNode.isFile() && topNode.asFile().isWithSnapshot()) { } else if (topNode.isFile() && topNode.asFile().isWithSnapshot()) {
INodeFile file = topNode.asFile(); INodeFile file = topNode.asFile();
counts.add(file.getDiffs().deleteSnapshotDiff(bsps, post, prior, file, counts.add(file.getDiffs().deleteSnapshotDiff(reclaimContext, post, prior, file));
collectedBlocks, removedINodes));
} else if (topNode.isDirectory()) { } else if (topNode.isDirectory()) {
INodeDirectory dir = topNode.asDirectory(); INodeDirectory dir = topNode.asDirectory();
ChildrenDiff priorChildrenDiff = null; ChildrenDiff priorChildrenDiff = null;
@ -505,8 +485,8 @@ private static QuotaCounts cleanDeletedINode(
DirectoryDiff priorDiff = sf.getDiffs().getDiffById(prior); DirectoryDiff priorDiff = sf.getDiffs().getDiffById(prior);
if (priorDiff != null && priorDiff.getSnapshotId() == prior) { if (priorDiff != null && priorDiff.getSnapshotId() == prior) {
priorChildrenDiff = priorDiff.getChildrenDiff(); priorChildrenDiff = priorDiff.getChildrenDiff();
counts.add(priorChildrenDiff.destroyCreatedList(bsps, dir, counts.add(priorChildrenDiff.destroyCreatedList(reclaimContext,
collectedBlocks, removedINodes, removedUCFiles)); dir));
} }
} }
@ -636,13 +616,11 @@ public INode saveChild2Snapshot(INodeDirectory currentINode,
return child; return child;
} }
public void clear(BlockStoragePolicySuite bsps, INodeDirectory currentINode, public void clear(
final BlocksMapUpdateInfo collectedBlocks, final List<INode> INode.ReclaimContext reclaimContext, INodeDirectory currentINode) {
removedINodes, final List<Long> removedUCFiles) {
// destroy its diff list // destroy its diff list
for (DirectoryDiff diff : diffs) { for (DirectoryDiff diff : diffs) {
diff.destroyDiffAndCollectBlocks(bsps, currentINode, collectedBlocks, diff.destroyDiffAndCollectBlocks(reclaimContext, currentINode);
removedINodes);
} }
diffs.clear(); diffs.clear();
} }
@ -729,9 +707,8 @@ boolean computeDiffBetweenSnapshots(Snapshot fromSnapshot,
} }
public QuotaCounts cleanDirectory( public QuotaCounts cleanDirectory(
final BlockStoragePolicySuite bsps, final INodeDirectory currentINode, INode.ReclaimContext reclaimContext, final INodeDirectory currentINode,
final int snapshot, int prior, final BlocksMapUpdateInfo collectedBlocks, final int snapshot, int prior) {
final List<INode> removedINodes, List<Long> removedUCFiles) {
QuotaCounts counts = new QuotaCounts.Builder().build(); QuotaCounts counts = new QuotaCounts.Builder().build();
Map<INode, INode> priorCreated = null; Map<INode, INode> priorCreated = null;
Map<INode, INode> priorDeleted = null; Map<INode, INode> priorDeleted = null;
@ -740,11 +717,11 @@ public QuotaCounts cleanDirectory(
// delete everything in created list // delete everything in created list
DirectoryDiff lastDiff = diffs.getLast(); DirectoryDiff lastDiff = diffs.getLast();
if (lastDiff != null) { if (lastDiff != null) {
counts.add(lastDiff.diff.destroyCreatedList(bsps, currentINode, counts.add(lastDiff.diff.destroyCreatedList(reclaimContext,
collectedBlocks, removedINodes, removedUCFiles)); currentINode));
} }
counts.add(currentINode.cleanSubtreeRecursively(bsps, snapshot, prior, counts.add(currentINode.cleanSubtreeRecursively(reclaimContext,
collectedBlocks, removedINodes, removedUCFiles, priorDeleted)); snapshot, prior, priorDeleted));
} else { } else {
// update prior // update prior
prior = getDiffs().updatePrior(snapshot, prior); prior = getDiffs().updatePrior(snapshot, prior);
@ -760,10 +737,10 @@ public QuotaCounts cleanDirectory(
} }
} }
counts.add(getDiffs().deleteSnapshotDiff(bsps, snapshot, prior, counts.add(getDiffs().deleteSnapshotDiff(reclaimContext, snapshot, prior,
currentINode, collectedBlocks, removedINodes)); currentINode));
counts.add(currentINode.cleanSubtreeRecursively(bsps, snapshot, prior, counts.add(currentINode.cleanSubtreeRecursively(reclaimContext,
collectedBlocks, removedINodes, removedUCFiles, priorDeleted)); snapshot, prior, priorDeleted));
// check priorDiff again since it may be created during the diff deletion // check priorDiff again since it may be created during the diff deletion
if (prior != Snapshot.NO_SNAPSHOT_ID) { if (prior != Snapshot.NO_SNAPSHOT_ID) {
@ -779,8 +756,8 @@ public QuotaCounts cleanDirectory(
for (INode cNode : priorDiff.getChildrenDiff().getList( for (INode cNode : priorDiff.getChildrenDiff().getList(
ListType.CREATED)) { ListType.CREATED)) {
if (priorCreated.containsKey(cNode)) { if (priorCreated.containsKey(cNode)) {
counts.add(cNode.cleanSubtree(bsps, snapshot, Snapshot.NO_SNAPSHOT_ID, counts.add(cNode.cleanSubtree(reclaimContext,
collectedBlocks, removedINodes, removedUCFiles)); snapshot, Snapshot.NO_SNAPSHOT_ID));
} }
} }
} }
@ -796,8 +773,8 @@ public QuotaCounts cleanDirectory(
for (INode dNode : priorDiff.getChildrenDiff().getList( for (INode dNode : priorDiff.getChildrenDiff().getList(
ListType.DELETED)) { ListType.DELETED)) {
if (priorDeleted == null || !priorDeleted.containsKey(dNode)) { if (priorDeleted == null || !priorDeleted.containsKey(dNode)) {
counts.add(cleanDeletedINode(bsps, dNode, snapshot, prior, counts.add(cleanDeletedINode(reclaimContext,
collectedBlocks, removedINodes, removedUCFiles)); dNode, snapshot, prior));
} }
} }
} }

18
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiff.java
View File

@ -20,10 +20,8 @@
import java.io.DataOutput; import java.io.DataOutput;
import java.io.IOException; import java.io.IOException;
import java.util.Arrays; import java.util.Arrays;
import java.util.List;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization; import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization;
import org.apache.hadoop.hdfs.server.namenode.INode; import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo; import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
@ -82,13 +80,12 @@ public BlockInfoContiguous[] getBlocks() {
@Override @Override
QuotaCounts combinePosteriorAndCollectBlocks( QuotaCounts combinePosteriorAndCollectBlocks(
BlockStoragePolicySuite bsps, INodeFile currentINode, INode.ReclaimContext reclaimContext, INodeFile currentINode,
FileDiff posterior, BlocksMapUpdateInfo collectedBlocks, FileDiff posterior) {
final List<INode> removedINodes) {
FileWithSnapshotFeature sf = currentINode.getFileWithSnapshotFeature(); FileWithSnapshotFeature sf = currentINode.getFileWithSnapshotFeature();
assert sf != null : "FileWithSnapshotFeature is null"; assert sf != null : "FileWithSnapshotFeature is null";
return sf.updateQuotaAndCollectBlocks( return sf.updateQuotaAndCollectBlocks(reclaimContext,
bsps, currentINode, posterior, collectedBlocks, removedINodes); currentINode, posterior);
} }
@Override @Override
@ -112,11 +109,10 @@ void write(DataOutput out, ReferenceMap referenceMap) throws IOException {
} }
@Override @Override
QuotaCounts destroyDiffAndCollectBlocks(BlockStoragePolicySuite bsps, INodeFile currentINode, QuotaCounts destroyDiffAndCollectBlocks(
BlocksMapUpdateInfo collectedBlocks, final List<INode> removedINodes) { INode.ReclaimContext reclaimContext, INodeFile currentINode) {
return currentINode.getFileWithSnapshotFeature() return currentINode.getFileWithSnapshotFeature()
.updateQuotaAndCollectBlocks(bsps, currentINode, this, collectedBlocks, .updateQuotaAndCollectBlocks(reclaimContext, currentINode, this);
removedINodes);
} }
public void destroyAndCollectSnapshotBlocks( public void destroyAndCollectSnapshotBlocks(

11
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java
View File

@ -23,7 +23,6 @@
import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.namenode.INode; import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo; import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
@ -97,16 +96,14 @@ public BlockInfoContiguous[] findLaterSnapshotBlocks(int snapshotId) {
* up to the file length of the latter. * up to the file length of the latter.
* Collect unused blocks of the removed snapshot. * Collect unused blocks of the removed snapshot.
*/ */
void combineAndCollectSnapshotBlocks(BlockStoragePolicySuite bsps, INodeFile file, void combineAndCollectSnapshotBlocks(
FileDiff removed, INode.ReclaimContext reclaimContext, INodeFile file, FileDiff removed) {
BlocksMapUpdateInfo collectedBlocks,
List<INode> removedINodes) {
BlockInfoContiguous[] removedBlocks = removed.getBlocks(); BlockInfoContiguous[] removedBlocks = removed.getBlocks();
if(removedBlocks == null) { if(removedBlocks == null) {
FileWithSnapshotFeature sf = file.getFileWithSnapshotFeature(); FileWithSnapshotFeature sf = file.getFileWithSnapshotFeature();
assert sf != null : "FileWithSnapshotFeature is null"; assert sf != null : "FileWithSnapshotFeature is null";
if(sf.isCurrentFileDeleted()) if(sf.isCurrentFileDeleted())
sf.collectBlocksAndClear(bsps, file, collectedBlocks, removedINodes); sf.collectBlocksAndClear(reclaimContext, file);
return; return;
} }
int p = getPrior(removed.getSnapshotId(), true); int p = getPrior(removed.getSnapshotId(), true);
@ -139,7 +136,7 @@ void combineAndCollectSnapshotBlocks(BlockStoragePolicySuite bsps, INodeFile fil
// Collect the remaining blocks of the file, ignoring truncate block // Collect the remaining blocks of the file, ignoring truncate block
for(;i < removedBlocks.length; i++) { for(;i < removedBlocks.length; i++) {
if(dontRemoveBlock == null || !removedBlocks[i].equals(dontRemoveBlock)) { if(dontRemoveBlock == null || !removedBlocks[i].equals(dontRemoveBlock)) {
collectedBlocks.addDeleteBlock(removedBlocks[i]); reclaimContext.collectedBlocks().addDeleteBlock(removedBlocks[i]);
} }
} }
} }

35
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
View File

@ -23,10 +23,8 @@
import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.server.namenode.AclFeature; import org.apache.hadoop.hdfs.server.namenode.AclFeature;
import org.apache.hadoop.hdfs.server.namenode.INode; import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
import org.apache.hadoop.hdfs.server.namenode.AclStorage; import org.apache.hadoop.hdfs.server.namenode.AclStorage;
import org.apache.hadoop.hdfs.server.namenode.INodeFile; import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.hdfs.server.namenode.INodeFileAttributes; import org.apache.hadoop.hdfs.server.namenode.INodeFileAttributes;
@ -118,22 +116,21 @@ public String getDetailedString() {
return (isCurrentFileDeleted()? "(DELETED), ": ", ") + diffs; return (isCurrentFileDeleted()? "(DELETED), ": ", ") + diffs;
} }
public QuotaCounts cleanFile(final BlockStoragePolicySuite bsps, public QuotaCounts cleanFile(INode.ReclaimContext reclaimContext,
final INodeFile file, final int snapshotId, final INodeFile file, final int snapshotId,
int priorSnapshotId, final BlocksMapUpdateInfo collectedBlocks, int priorSnapshotId) {
final List<INode> removedINodes) {
if (snapshotId == Snapshot.CURRENT_STATE_ID) { if (snapshotId == Snapshot.CURRENT_STATE_ID) {
// delete the current file while the file has snapshot feature // delete the current file while the file has snapshot feature
if (!isCurrentFileDeleted()) { if (!isCurrentFileDeleted()) {
file.recordModification(priorSnapshotId); file.recordModification(priorSnapshotId);
deleteCurrentFile(); deleteCurrentFile();
} }
collectBlocksAndClear(bsps, file, collectedBlocks, removedINodes); collectBlocksAndClear(reclaimContext, file);
return new QuotaCounts.Builder().build(); return new QuotaCounts.Builder().build();
} else { // delete the snapshot } else { // delete the snapshot
priorSnapshotId = getDiffs().updatePrior(snapshotId, priorSnapshotId); priorSnapshotId = getDiffs().updatePrior(snapshotId, priorSnapshotId);
return diffs.deleteSnapshotDiff(bsps, snapshotId, priorSnapshotId, file, return diffs.deleteSnapshotDiff(reclaimContext,
collectedBlocks, removedINodes); snapshotId, priorSnapshotId, file);
} }
} }
@ -141,14 +138,12 @@ public void clearDiffs() {
this.diffs.clear(); this.diffs.clear();
} }
public QuotaCounts updateQuotaAndCollectBlocks(BlockStoragePolicySuite bsps, INodeFile file, public QuotaCounts updateQuotaAndCollectBlocks(
FileDiff removed, BlocksMapUpdateInfo collectedBlocks, INode.ReclaimContext reclaimContext, INodeFile file, FileDiff removed) {
final List<INode> removedINodes) {
byte storagePolicyID = file.getStoragePolicyID(); byte storagePolicyID = file.getStoragePolicyID();
BlockStoragePolicy bsp = null; BlockStoragePolicy bsp = null;
if (storagePolicyID != HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED) { if (storagePolicyID != HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED) {
bsp = bsps.getPolicy(file.getStoragePolicyID()); bsp = reclaimContext.storagePolicySuite().getPolicy(file.getStoragePolicyID());
} }
@ -180,8 +175,7 @@ public QuotaCounts updateQuotaAndCollectBlocks(BlockStoragePolicySuite bsps, INo
} }
} }
getDiffs().combineAndCollectSnapshotBlocks( getDiffs().combineAndCollectSnapshotBlocks(reclaimContext, file, removed);
bsps, file, removed, collectedBlocks, removedINodes);
QuotaCounts current = file.storagespaceConsumed(bsp); QuotaCounts current = file.storagespaceConsumed(bsp);
oldCounts.subtract(current); oldCounts.subtract(current);
@ -192,11 +186,11 @@ public QuotaCounts updateQuotaAndCollectBlocks(BlockStoragePolicySuite bsps, INo
* If some blocks at the end of the block list no longer belongs to * If some blocks at the end of the block list no longer belongs to
* any inode, collect them and update the block list. * any inode, collect them and update the block list.
*/ */
public void collectBlocksAndClear(final BlockStoragePolicySuite bsps, final INodeFile file, public void collectBlocksAndClear(
final BlocksMapUpdateInfo info, final List<INode> removedINodes) { INode.ReclaimContext reclaimContext, final INodeFile file) {
// check if everything is deleted. // check if everything is deleted.
if (isCurrentFileDeleted() && getDiffs().asList().isEmpty()) { if (isCurrentFileDeleted() && getDiffs().asList().isEmpty()) {
file.destroyAndCollectBlocks(bsps, info, removedINodes, null); file.destroyAndCollectBlocks(reclaimContext);
return; return;
} }
// find max file size. // find max file size.
@ -212,8 +206,9 @@ public void collectBlocksAndClear(final BlockStoragePolicySuite bsps, final INod
FileDiff last = diffs.getLast(); FileDiff last = diffs.getLast();
BlockInfoContiguous[] snapshotBlocks = last == null ? null : last.getBlocks(); BlockInfoContiguous[] snapshotBlocks = last == null ? null : last.getBlocks();
if(snapshotBlocks == null) if(snapshotBlocks == null)
file.collectBlocksBeyondMax(max, info); file.collectBlocksBeyondMax(max, reclaimContext.collectedBlocks());
else else
file.collectBlocksBeyondSnapshot(snapshotBlocks, info); file.collectBlocksBeyondSnapshot(snapshotBlocks,
reclaimContext.collectedBlocks());
} }
} }

6
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
View File

@ -235,8 +235,10 @@ public void deleteSnapshot(final INodesInPath iip, final String snapshotName,
BlocksMapUpdateInfo collectedBlocks, final List<INode> removedINodes) BlocksMapUpdateInfo collectedBlocks, final List<INode> removedINodes)
throws IOException { throws IOException {
INodeDirectory srcRoot = getSnapshottableRoot(iip); INodeDirectory srcRoot = getSnapshottableRoot(iip);
srcRoot.removeSnapshot(fsdir.getBlockStoragePolicySuite(), snapshotName, srcRoot.removeSnapshot(
collectedBlocks, removedINodes); new INode.ReclaimContext(fsdir.getBlockStoragePolicySuite(),
collectedBlocks, removedINodes, null),
snapshotName);
numSnapshots.getAndDecrement(); numSnapshots.getAndDecrement();
} }

8
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestFileWithSnapshotFeature.java
View File

@ -62,8 +62,9 @@ public void testUpdateQuotaAndCollectBlocks() {
INode.BlocksMapUpdateInfo collectedBlocks = mock( INode.BlocksMapUpdateInfo collectedBlocks = mock(
INode.BlocksMapUpdateInfo.class); INode.BlocksMapUpdateInfo.class);
ArrayList<INode> removedINodes = new ArrayList<>(); ArrayList<INode> removedINodes = new ArrayList<>();
QuotaCounts counts = sf.updateQuotaAndCollectBlocks( INode.ReclaimContext ctx = new INode.ReclaimContext(
bsps, file, diff, collectedBlocks, removedINodes); bsps, collectedBlocks, removedINodes, null);
QuotaCounts counts = sf.updateQuotaAndCollectBlocks(ctx, file, diff);
Assert.assertEquals(0, counts.getStorageSpace()); Assert.assertEquals(0, counts.getStorageSpace());
Assert.assertTrue(counts.getTypeSpaces().allLessOrEqual(0)); Assert.assertTrue(counts.getTypeSpaces().allLessOrEqual(0));
@ -78,8 +79,7 @@ public void testUpdateQuotaAndCollectBlocks() {
.thenReturn(Lists.newArrayList(SSD)); .thenReturn(Lists.newArrayList(SSD));
when(bsp.chooseStorageTypes(REPL_3)) when(bsp.chooseStorageTypes(REPL_3))
.thenReturn(Lists.newArrayList(DISK)); .thenReturn(Lists.newArrayList(DISK));
counts = sf.updateQuotaAndCollectBlocks( counts = sf.updateQuotaAndCollectBlocks(ctx, file, diff);
bsps, file, diff, collectedBlocks, removedINodes);
Assert.assertEquals((REPL_3 - REPL_1) * BLOCK_SIZE, Assert.assertEquals((REPL_3 - REPL_1) * BLOCK_SIZE,
counts.getStorageSpace()); counts.getStorageSpace());
Assert.assertEquals(BLOCK_SIZE, counts.getTypeSpaces().get(DISK)); Assert.assertEquals(BLOCK_SIZE, counts.getTypeSpaces().get(DISK));