From c90e1905072a75141585c91bd4687b2babe09b41 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Wed, 24 May 2017 17:21:22 -0700 Subject: [PATCH] Revert "HDFS-10797. Disk usage summary of snapshots causes renamed blocks to get counted twice. Contributed by Sean Mackrory." This reverts commit 6a38d118d86b7907009bcec34f1b788d076f1d1c. --- .../ContentSummaryComputationContext.java | 94 +-------- .../hadoop/hdfs/server/namenode/INode.java | 1 - .../hdfs/server/namenode/INodeDirectory.java | 11 +- .../hdfs/server/namenode/INodeFile.java | 1 - .../hdfs/server/namenode/INodeReference.java | 2 - .../hdfs/server/namenode/INodeSymlink.java | 1 - .../DirectorySnapshottableFeature.java | 9 +- .../DirectoryWithSnapshotFeature.java | 14 +- .../server/namenode/snapshot/Snapshot.java | 1 - .../snapshot/TestRenameWithSnapshots.java | 199 ------------------ 10 files changed, 26 insertions(+), 307 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java index 53fa552077e..b35270d3534 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java @@ -21,10 +21,6 @@ import com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite; -import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; - -import java.util.HashSet; -import java.util.Set; @InterfaceAudience.Private @InterfaceStability.Unstable @@ -39,8 +35,6 @@ public class ContentSummaryComputationContext { private long yieldCount = 0; private long sleepMilliSec = 0; private int sleepNanoSec = 0; - private Set includedNodes = new HashSet<>(); - private Set deletedSnapshottedNodes = new HashSet<>(); /** * Constructor @@ -57,8 +51,8 @@ public class ContentSummaryComputationContext { this.fsn = fsn; this.limitPerRun = limitPerRun; this.nextCountLimit = limitPerRun; - setCounts(new ContentCounts.Builder().build()); - setSnapshotCounts(new ContentCounts.Builder().build()); + this.counts = new ContentCounts.Builder().build(); + this.snapshotCounts = new ContentCounts.Builder().build(); this.sleepMilliSec = sleepMicroSec/1000; this.sleepNanoSec = (int)((sleepMicroSec%1000)*1000); } @@ -88,7 +82,6 @@ public class ContentSummaryComputationContext { } // Have we reached the limit? - ContentCounts counts = getCounts(); long currentCount = counts.getFileCount() + counts.getSymlinkCount() + counts.getDirectoryCount() + @@ -130,22 +123,14 @@ public class ContentSummaryComputationContext { } /** Get the content counts */ - public synchronized ContentCounts getCounts() { + public ContentCounts getCounts() { return counts; } - private synchronized void setCounts(ContentCounts counts) { - this.counts = counts; - } - public ContentCounts getSnapshotCounts() { return snapshotCounts; } - private void setSnapshotCounts(ContentCounts snapshotCounts) { - this.snapshotCounts = snapshotCounts; - } - public BlockStoragePolicySuite getBlockStoragePolicySuite() { Preconditions.checkState((bsps != null || fsn != null), "BlockStoragePolicySuite must be either initialized or available via" + @@ -153,77 +138,4 @@ public class ContentSummaryComputationContext { return (bsps != null) ? bsps: fsn.getBlockManager().getStoragePolicySuite(); } - - /** - * If the node is an INodeReference, resolves it to the actual inode. - * Snapshot diffs represent renamed / moved files as different - * INodeReferences, but the underlying INode it refers to is consistent. - * - * @param node - * @return The referred INode if there is one, else returns the input - * unmodified. - */ - private INode resolveINodeReference(INode node) { - if (node.isReference() && node instanceof INodeReference) { - return ((INodeReference)node).getReferredINode(); - } - return node; - } - - /** - * Reports that a node is about to be included in this summary. Can be used - * either to simply report that a node has been including, or check whether - * a node has already been included. - * - * @param node - * @return true if node has already been included - */ - public boolean nodeIncluded(INode node) { - INode resolvedNode = resolveINodeReference(node); - synchronized (includedNodes) { - if (!includedNodes.contains(resolvedNode)) { - includedNodes.add(resolvedNode); - return false; - } - } - return true; - } - - /** - * Schedules a node that is listed as DELETED in a snapshot's diff to be - * included in the summary at the end of computation. See - * {@link #tallyDeletedSnapshottedINodes()} for more context. - * - * @param node - */ - public void reportDeletedSnapshottedNode(INode node) { - deletedSnapshottedNodes.add(node); - } - - /** - * Finalizes the computation by including all nodes that were reported as - * deleted by a snapshot but have not been already including due to other - * references. - *

- * Nodes that get renamed are listed in the snapshot's diff as both DELETED - * under the old name and CREATED under the new name. The computation - * relies on nodes to report themselves as being included (via - * {@link #nodeIncluded(INode)} as the only reliable way to determine which - * nodes were renamed within the tree being summarized and which were - * removed (either by deletion or being renamed outside of the tree). - */ - public synchronized void tallyDeletedSnapshottedINodes() { - /* Temporarily create a new counts object so these results can then be - added to both counts and snapshotCounts */ - ContentCounts originalCounts = getCounts(); - setCounts(new ContentCounts.Builder().build()); - for (INode node : deletedSnapshottedNodes) { - if (!nodeIncluded(node)) { - node.computeContentSummary(Snapshot.CURRENT_STATE_ID, this); - } - } - originalCounts.addContents(getCounts()); - snapshotCounts.addContents(getCounts()); - setCounts(originalCounts); - } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java index e1db990c1f2..c6258a173b3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java @@ -429,7 +429,6 @@ public abstract class INode implements INodeAttributes, Diff.Element { public final ContentSummary computeAndConvertContentSummary(int snapshotId, ContentSummaryComputationContext summary) { computeContentSummary(snapshotId, summary); - summary.tallyDeletedSnapshottedINodes(); final ContentCounts counts = summary.getCounts(); final ContentCounts snapshotCounts = summary.getSnapshotCounts(); final QuotaCounts q = getQuotaCounts(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java index a4098bc0553..a29a1187018 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java @@ -630,10 +630,17 @@ public class INodeDirectory extends INodeWithAdditionalFields @Override public ContentSummaryComputationContext computeContentSummary(int snapshotId, ContentSummaryComputationContext summary) { - summary.nodeIncluded(this); final DirectoryWithSnapshotFeature sf = getDirectoryWithSnapshotFeature(); if (sf != null && snapshotId == Snapshot.CURRENT_STATE_ID) { - sf.computeContentSummary4Snapshot(summary); + final ContentCounts counts = new ContentCounts.Builder().build(); + // if the getContentSummary call is against a non-snapshot path, the + // computation should include all the deleted files/directories + sf.computeContentSummary4Snapshot(summary.getBlockStoragePolicySuite(), + counts); + summary.getCounts().addContents(counts); + // Also add ContentSummary to snapshotCounts (So we can extract it + // later from the ContentSummary of all). + summary.getSnapshotCounts().addContents(counts); } final DirectoryWithQuotaFeature q = getDirectoryWithQuotaFeature(); if (q != null && snapshotId == Snapshot.CURRENT_STATE_ID) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java index 9cdac06cc81..fcdb064fd66 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java @@ -840,7 +840,6 @@ public class INodeFile extends INodeWithAdditionalFields @Override public final ContentSummaryComputationContext computeContentSummary( int snapshotId, final ContentSummaryComputationContext summary) { - summary.nodeIncluded(this); final ContentCounts counts = summary.getCounts(); counts.addContent(Content.FILE, 1); final long fileLen = computeFileSize(snapshotId); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java index 56aaf8ce058..1b852374c5d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java @@ -315,7 +315,6 @@ public abstract class INodeReference extends INode { @Override public ContentSummaryComputationContext computeContentSummary(int snapshotId, ContentSummaryComputationContext summary) { - summary.nodeIncluded(this); return referred.computeContentSummary(snapshotId, summary); } @@ -505,7 +504,6 @@ public abstract class INodeReference extends INode { @Override public final ContentSummaryComputationContext computeContentSummary( int snapshotId, ContentSummaryComputationContext summary) { - summary.nodeIncluded(this); final int s = snapshotId < lastSnapshotId ? snapshotId : lastSnapshotId; // only count storagespace for WithName final QuotaCounts q = computeQuotaUsage( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java index 1223f4ef86c..c76bea090f1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java @@ -96,7 +96,6 @@ public class INodeSymlink extends INodeWithAdditionalFields { @Override public ContentSummaryComputationContext computeContentSummary(int snapshotId, final ContentSummaryComputationContext summary) { - summary.nodeIncluded(this); summary.getCounts().addContent(Content.SYMLINK, 1); return summary; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java index 3039ad3486f..fbfc278703d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java @@ -30,9 +30,9 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.protocol.QuotaExceededException; import org.apache.hadoop.hdfs.protocol.SnapshotException; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite; import org.apache.hadoop.hdfs.server.namenode.Content; import org.apache.hadoop.hdfs.server.namenode.ContentCounts; -import org.apache.hadoop.hdfs.server.namenode.ContentSummaryComputationContext; import org.apache.hadoop.hdfs.server.namenode.INode; import org.apache.hadoop.hdfs.server.namenode.INodeDirectory; import org.apache.hadoop.hdfs.server.namenode.INodeDirectory.SnapshotAndINode; @@ -233,12 +233,11 @@ public class DirectorySnapshottableFeature extends DirectoryWithSnapshotFeature } @Override - public void computeContentSummary4Snapshot(ContentSummaryComputationContext - context) { - ContentCounts counts = context.getCounts(); + public void computeContentSummary4Snapshot(final BlockStoragePolicySuite bsps, + final ContentCounts counts) { counts.addContent(Content.SNAPSHOT, snapshotsByNames.size()); counts.addContent(Content.SNAPSHOTTABLE_DIRECTORY, 1); - super.computeContentSummary4Snapshot(context); + super.computeContentSummary4Snapshot(bsps, counts); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java index 9addbfa7ace..0111b3bfb65 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java @@ -30,6 +30,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.protocol.QuotaExceededException; import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite; import org.apache.hadoop.hdfs.server.namenode.AclStorage; +import org.apache.hadoop.hdfs.server.namenode.ContentCounts; import org.apache.hadoop.hdfs.server.namenode.ContentSummaryComputationContext; import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization; import org.apache.hadoop.hdfs.server.namenode.INode; @@ -628,13 +629,18 @@ public class DirectoryWithSnapshotFeature implements INode.Feature { return counts; } - public void computeContentSummary4Snapshot( - ContentSummaryComputationContext context) { + public void computeContentSummary4Snapshot(final BlockStoragePolicySuite bsps, + final ContentCounts counts) { + // Create a new blank summary context for blocking processing of subtree. + ContentSummaryComputationContext summary = + new ContentSummaryComputationContext(bsps); for(DirectoryDiff d : diffs) { - for(INode deletedNode : d.getChildrenDiff().getList(ListType.DELETED)) { - context.reportDeletedSnapshottedNode(deletedNode); + for(INode deleted : d.getChildrenDiff().getList(ListType.DELETED)) { + deleted.computeContentSummary(Snapshot.CURRENT_STATE_ID, summary); } } + // Add the counts from deleted trees. + counts.addContents(summary.getCounts()); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java index 832a3391f5b..e98e766d34a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/Snapshot.java @@ -177,7 +177,6 @@ public class Snapshot implements Comparable { @Override public ContentSummaryComputationContext computeContentSummary( int snapshotId, ContentSummaryComputationContext summary) { - summary.nodeIncluded(this); return computeDirectoryContentSummary(summary, snapshotId); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java index d1b3aa6f870..91eec7884d9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java @@ -36,10 +36,8 @@ import java.util.Random; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Options.Rename; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; @@ -2413,201 +2411,4 @@ public class TestRenameWithSnapshots { assertTrue(existsInDiffReport(entries, DiffType.RENAME, "foo/file2", "newDir/file2")); assertTrue(existsInDiffReport(entries, DiffType.RENAME, "foo/file3", "newDir/file1")); } - - private void checkSpaceConsumed(String message, Path directory, - long expectedSpace) throws Exception { - ContentSummary summary = hdfs.getContentSummary(directory); - assertEquals(message, expectedSpace, summary.getSpaceConsumed()); - } - - /** - * Runs through various combinations of renames, deletes, appends and other - * operations in a snapshotted directory and ensures disk usage summaries - * (e.g. du -s) are computed correctly. - * - * @throws Exception - */ - @Test (timeout=300000) - public void testDu() throws Exception { - File tempFile = File.createTempFile("testDu-", ".tmp"); - tempFile.deleteOnExit(); - - final FileSystem localfs = FileSystem.getLocal(conf); - final Path localOriginal = new Path(tempFile.getPath()); - final Path dfsRoot = new Path("/testDu"); - final Path dfsOriginal = new Path(dfsRoot, "original"); - final Path dfsRenamed1 = new Path(dfsRoot, "renamed1"); - final Path dfsRenamed2 = new Path(dfsRoot, "renamed2"); - final Path dfsAppended = new Path(dfsRoot, "appended"); - - /* We will test with a single block worth of data. If we don't at least use - a multiple of BLOCKSIZE, append operations will modify snapshotted blocks - and other factors will come into play here that we'll have to account for */ - final long spaceIncrement = BLOCKSIZE * REPL; - final byte[] appendData = new byte[(int) BLOCKSIZE]; - DFSTestUtil.createFile(localfs, localOriginal, BLOCKSIZE, REPL, SEED); - - FSDataOutputStream out = null; - long expectedSpace = 0; - - hdfs.mkdirs(dfsRoot); - checkSpaceConsumed("Du is wrong immediately", - dfsRoot, 0L); - - hdfs.copyFromLocalFile(localOriginal, dfsOriginal); - expectedSpace += spaceIncrement; - checkSpaceConsumed("Du is wrong after creating / copying file", - dfsRoot, expectedSpace); - - SnapshotTestHelper.createSnapshot(hdfs, dfsRoot, "s0"); - checkSpaceConsumed("Du is wrong after snapshotting", - dfsRoot, expectedSpace); - - hdfs.rename(dfsOriginal, dfsRenamed1); - checkSpaceConsumed("Du is wrong after 1 rename", - dfsRoot, expectedSpace); - - hdfs.rename(dfsRenamed1, dfsRenamed2); - checkSpaceConsumed("Du is wrong after 2 renames", - dfsRoot, expectedSpace); - - hdfs.delete(dfsRenamed2, false); - checkSpaceConsumed("Du is wrong after deletion", - dfsRoot, expectedSpace); - - hdfs.copyFromLocalFile(localOriginal, dfsOriginal); - expectedSpace += spaceIncrement; - checkSpaceConsumed("Du is wrong after replacing a renamed file", - dfsRoot, expectedSpace); - - hdfs.copyFromLocalFile(localOriginal, dfsAppended); - expectedSpace += spaceIncrement; - SnapshotTestHelper.createSnapshot(hdfs, dfsRoot, "s1"); - - out = hdfs.append(dfsAppended); - out.write(appendData); - out.close(); - expectedSpace += spaceIncrement; - checkSpaceConsumed("Du is wrong after 1 snapshot + append", - dfsRoot, expectedSpace); - - SnapshotTestHelper.createSnapshot(hdfs, dfsRoot, "s2"); - out = hdfs.append(dfsAppended); - out.write(appendData); - out.close(); - expectedSpace += spaceIncrement; - checkSpaceConsumed("Du is wrong after 2 snapshot + appends", - dfsRoot, expectedSpace); - - SnapshotTestHelper.createSnapshot(hdfs, dfsRoot, "s3"); - out = hdfs.append(dfsAppended); - out.write(appendData); - out.close(); - expectedSpace += spaceIncrement; - hdfs.rename(dfsAppended, dfsRenamed1); - checkSpaceConsumed("Du is wrong after snapshot, append, & rename", - dfsRoot, expectedSpace); - hdfs.delete(dfsRenamed1, false); - // everything but the last append is snapshotted - expectedSpace -= spaceIncrement; - checkSpaceConsumed("Du is wrong after snapshot, append, delete & rename", - dfsRoot, expectedSpace); - - hdfs.delete(dfsOriginal, false); - hdfs.deleteSnapshot(dfsRoot, "s0"); - hdfs.deleteSnapshot(dfsRoot, "s1"); - hdfs.deleteSnapshot(dfsRoot, "s2"); - hdfs.deleteSnapshot(dfsRoot, "s3"); - expectedSpace = 0; - checkSpaceConsumed("Du is wrong after deleting all files and snapshots", - dfsRoot, expectedSpace); - } - - /** - * Runs through various combinations of renames, deletes, appends and other - * operations between two snapshotted directories and ensures disk usage - * summaries (e.g. du -s) are computed correctly. - * - * This test currently assumes some incorrect behavior when files have been - * moved between subdirectories of the one being queried. In the cases - * below, only 1 block worth of data should ever actually be used. However - * if there are 2 - 3 subdirectories that do contained or have contained - * when snapshotted the same file, that file will be counted 2-3 times, - * respectively, since each directory is computed independently recursively. - * - * @throws Exception - */ - @Test (timeout=300000) - public void testDuMultipleDirs() throws Exception { - File tempFile = File.createTempFile("testDuMultipleDirs-", "" + ".tmp"); - tempFile.deleteOnExit(); - - final FileSystem localfs = FileSystem.getLocal(conf); - final Path localOriginal = new Path(tempFile.getPath()); - final Path dfsRoot = new Path("/testDuMultipleDirs"); - final Path snapshottable1 = new Path(dfsRoot, "snapshottable1"); - final Path snapshottable2 = new Path(dfsRoot, "snapshottable2"); - final Path nonsnapshottable = new Path(dfsRoot, "nonsnapshottable"); - final Path subdirectory = new Path(snapshottable1, "subdirectory"); - final Path dfsOriginal = new Path(snapshottable1, "file"); - final Path renamedNonsnapshottable = new Path(nonsnapshottable, "file"); - final Path renamedSnapshottable = new Path(snapshottable2, "file"); - final Path renamedSubdirectory = new Path(subdirectory, "file"); - - /* We will test with a single block worth of data. If we don't at least use - a multiple of BLOCKSIZE, append operations will modify snapshotted blocks - and other factors will come into play here that we'll have to account for */ - final long spaceConsumed = BLOCKSIZE * REPL; - DFSTestUtil.createFile(localfs, localOriginal, BLOCKSIZE, REPL, SEED); - - hdfs.mkdirs(snapshottable1); - hdfs.mkdirs(snapshottable2); - hdfs.mkdirs(nonsnapshottable); - hdfs.mkdirs(subdirectory); - checkSpaceConsumed("Du is wrong immediately", - dfsRoot, 0L); - - hdfs.copyFromLocalFile(localOriginal, dfsOriginal); - checkSpaceConsumed("Du is wrong after creating / copying file", - snapshottable1, spaceConsumed); - - SnapshotTestHelper.createSnapshot(hdfs, snapshottable1, "s1"); - checkSpaceConsumed("Du is wrong in original dir after 1st snapshot", - snapshottable1, spaceConsumed); - - hdfs.rename(dfsOriginal, renamedNonsnapshottable); - checkSpaceConsumed("Du is wrong in original dir after 1st rename", - snapshottable1, spaceConsumed); - checkSpaceConsumed("Du is wrong in non-snapshottable dir after 1st rename", - nonsnapshottable, spaceConsumed); - checkSpaceConsumed("Du is wrong in root dir after 1st rename", - dfsRoot, spaceConsumed); - - hdfs.rename(renamedNonsnapshottable, renamedSnapshottable); - checkSpaceConsumed("Du is wrong in original dir after 2nd rename", - snapshottable1, spaceConsumed); - checkSpaceConsumed("Du is wrong in non-snapshottable dir after 2nd rename", - nonsnapshottable, 0); - checkSpaceConsumed("Du is wrong in snapshottable dir after 2nd rename", - snapshottable2, spaceConsumed); - checkSpaceConsumed("Du is wrong in root dir after 2nd rename", - dfsRoot, spaceConsumed); - - SnapshotTestHelper.createSnapshot(hdfs, snapshottable2, "s2"); - hdfs.rename(renamedSnapshottable, renamedSubdirectory); - checkSpaceConsumed("Du is wrong in original dir after 3rd rename", - snapshottable1, spaceConsumed); - checkSpaceConsumed("Du is wrong in snapshottable dir after 3rd rename", - snapshottable2, spaceConsumed); - checkSpaceConsumed("Du is wrong in original subdirectory after 3rd rename", - subdirectory, spaceConsumed); - checkSpaceConsumed("Du is wrong in root dir after 3rd rename", - dfsRoot, spaceConsumed); - - hdfs.delete(renamedSubdirectory, false); - hdfs.deleteSnapshot(snapshottable1, "s1"); - hdfs.deleteSnapshot(snapshottable2, "s2"); - checkSpaceConsumed("Du is wrong after deleting all files and snapshots", - dfsRoot, 0); - } }