Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-8986. Add option to -du to calculate directory space usage excluding snapshots. Contributed by Xiao Chen.

This commit is contained in:
Wei-Chiu Chuang 2016-08-23 04:13:48 -07:00
parent dd76238a3b
commit f0efea490e
12 changed files with 432 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

119
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
View File

@ -34,6 +34,11 @@ public class ContentSummary extends QuotaUsage implements Writable{
private long length;
private long fileCount;
private long directoryCount;
// These fields are to track the snapshot-related portion of the values.
private long snapshotLength;
private long snapshotFileCount;
private long snapshotDirectoryCount;
private long snapshotSpaceConsumed;
/** We don't use generics. Instead override spaceConsumed and other methods
in order to keep backward compatibility. */
@ -56,6 +61,26 @@ public class ContentSummary extends QuotaUsage implements Writable{
return this;
}
public Builder snapshotLength(long snapshotLength) {
this.snapshotLength = snapshotLength;
return this;
}
public Builder snapshotFileCount(long snapshotFileCount) {
this.snapshotFileCount = snapshotFileCount;
return this;
}
public Builder snapshotDirectoryCount(long snapshotDirectoryCount) {
this.snapshotDirectoryCount = snapshotDirectoryCount;
return this;
}
public Builder snapshotSpaceConsumed(long snapshotSpaceConsumed) {
this.snapshotSpaceConsumed = snapshotSpaceConsumed;
return this;
}
@Override
public Builder quota(long quota){
super.quota(quota);
@ -107,6 +132,10 @@ public class ContentSummary extends QuotaUsage implements Writable{
private long length;
private long fileCount;
private long directoryCount;
private long snapshotLength;
private long snapshotFileCount;
private long snapshotDirectoryCount;
private long snapshotSpaceConsumed;
}
/** Constructor deprecated by ContentSummary.Builder*/
@ -142,17 +171,37 @@ public class ContentSummary extends QuotaUsage implements Writable{
this.length = builder.length;
this.fileCount = builder.fileCount;
this.directoryCount = builder.directoryCount;
this.snapshotLength = builder.snapshotLength;
this.snapshotFileCount = builder.snapshotFileCount;
this.snapshotDirectoryCount = builder.snapshotDirectoryCount;
this.snapshotSpaceConsumed = builder.snapshotSpaceConsumed;
}
/** @return the length */
public long getLength() {return length;}
public long getSnapshotLength() {
return snapshotLength;
}
/** @return the directory count */
public long getDirectoryCount() {return directoryCount;}
public long getSnapshotDirectoryCount() {
return snapshotDirectoryCount;
}
/** @return the file count */
public long getFileCount() {return fileCount;}
public long getSnapshotFileCount() {
return snapshotFileCount;
}
public long getSnapshotSpaceConsumed() {
return snapshotSpaceConsumed;
}
@Override
@InterfaceAudience.Private
public void write(DataOutput out) throws IOException {
@ -180,9 +229,14 @@ public class ContentSummary extends QuotaUsage implements Writable{
if (this == to) {
return true;
} else if (to instanceof ContentSummary) {
return getLength() == ((ContentSummary) to).getLength() &&
getFileCount() == ((ContentSummary) to).getFileCount() &&
getDirectoryCount() == ((ContentSummary) to).getDirectoryCount() &&
ContentSummary right = (ContentSummary) to;
return getLength() == right.getLength() &&
getFileCount() == right.getFileCount() &&
getDirectoryCount() == right.getDirectoryCount() &&
getSnapshotLength() == right.getSnapshotLength() &&
getSnapshotFileCount() == right.getSnapshotFileCount() &&
getSnapshotDirectoryCount() == right.getSnapshotDirectoryCount() &&
getSnapshotSpaceConsumed() == right.getSnapshotSpaceConsumed() &&
super.equals(to);
} else {
return super.equals(to);
@ -191,7 +245,9 @@ public class ContentSummary extends QuotaUsage implements Writable{
@Override
public int hashCode() {
long result = getLength() ^ getFileCount() ^ getDirectoryCount();
long result = getLength() ^ getFileCount() ^ getDirectoryCount()
^ getSnapshotLength() ^ getSnapshotFileCount()
^ getSnapshotDirectoryCount() ^ getSnapshotSpaceConsumed();
return ((int) result) ^ super.hashCode();
}
@ -255,15 +311,14 @@ public class ContentSummary extends QuotaUsage implements Writable{
* @param qOption a flag indicating if quota needs to be printed or not
* @return the string representation of the object
*/
@Override
public String toString(boolean qOption) {
return toString(qOption, false);
}
/** Return the string representation of the object in the output format.
* if qOption is false, output directory count, file count, and content size;
* if qOption is true, output quota and remaining quota as well.
* if hOption is false file sizes are returned in bytes
* if hOption is true file sizes are returned in human readable
* For description of the options,
* @see #toString(boolean, boolean, boolean, boolean, List)
*
* @param qOption a flag indicating if quota needs to be printed or not
* @param hOption a flag indicating if human readable output if to be used
@ -273,10 +328,24 @@ public class ContentSummary extends QuotaUsage implements Writable{
return toString(qOption, hOption, false, null);
}
/** Return the string representation of the object in the output format.
* For description of the options,
* @see #toString(boolean, boolean, boolean, boolean, List)
*
* @param qOption a flag indicating if quota needs to be printed or not
* @param hOption a flag indicating if human readable output is to be used
* @param xOption a flag indicating if calculation from snapshots is to be
* included in the output
* @return the string representation of the object
*/
public String toString(boolean qOption, boolean hOption, boolean xOption) {
return toString(qOption, hOption, false, xOption, null);
}
/**
* Return the string representation of the object in the output format.
* if tOption is true, display the quota by storage types,
* Otherwise, same logic with #toString(boolean,boolean)
* For description of the options,
* @see #toString(boolean, boolean, boolean, boolean, List)
*
* @param qOption a flag indicating if quota needs to be printed or not
* @param hOption a flag indicating if human readable output if to be used
@ -286,6 +355,29 @@ public class ContentSummary extends QuotaUsage implements Writable{
*/
public String toString(boolean qOption, boolean hOption,
boolean tOption, List<StorageType> types) {
return toString(qOption, hOption, tOption, false, types);
}
/** Return the string representation of the object in the output format.
* if qOption is false, output directory count, file count, and content size;
* if qOption is true, output quota and remaining quota as well.
* if hOption is false, file sizes are returned in bytes
* if hOption is true, file sizes are returned in human readable
* if tOption is true, display the quota by storage types
* if tOption is false, same logic with #toString(boolean,boolean)
* if xOption is false, output includes the calculation from snapshots
* if xOption is true, output excludes the calculation from snapshots
*
* @param qOption a flag indicating if quota needs to be printed or not
* @param hOption a flag indicating if human readable output is to be used
* @param tOption a flag indicating if display quota by storage types
* @param xOption a flag indicating if calculation from snapshots is to be
* included in the output
* @param types Storage types to display
* @return the string representation of the object
*/
public String toString(boolean qOption, boolean hOption, boolean tOption,
boolean xOption, List<StorageType> types) {
String prefix = "";
if (tOption) {
@ -296,11 +388,18 @@ public class ContentSummary extends QuotaUsage implements Writable{
prefix = getQuotaUsage(hOption);
}
if (xOption) {
return prefix + String.format(SUMMARY_FORMAT,
formatSize(directoryCount - snapshotDirectoryCount, hOption),
formatSize(fileCount - snapshotFileCount, hOption),
formatSize(length - snapshotLength, hOption));
} else {
return prefix + String.format(SUMMARY_FORMAT,
formatSize(directoryCount, hOption),
formatSize(fileCount, hOption),
formatSize(length, hOption));
}
}
/**
* Formats a size to be human readable or in bytes

20
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
View File

@ -51,6 +51,8 @@ public class Count extends FsCommand {
private static final String OPTION_HUMAN = "h";
private static final String OPTION_HEADER = "v";
private static final String OPTION_TYPE = "t";
// exclude snapshots from calculation. Only work on default columns.
private static final String OPTION_EXCLUDE_SNAPSHOT = "x";
//return the quota, namespace count and disk space usage.
private static final String OPTION_QUOTA_AND_USAGE = "u";
@ -58,7 +60,8 @@ public class Count extends FsCommand {
public static final String USAGE =
"[-" + OPTION_QUOTA + "] [-" + OPTION_HUMAN + "] [-" + OPTION_HEADER
+ "] [-" + OPTION_TYPE + " [<storage type>]] [-" +
OPTION_QUOTA_AND_USAGE + "] <path> ...";
OPTION_QUOTA_AND_USAGE + "] [-" + OPTION_EXCLUDE_SNAPSHOT
+ "] <path> ...";
public static final String DESCRIPTION =
"Count the number of directories, files and bytes under the paths\n" +
"that match the specified file pattern. The output columns are:\n" +
@ -72,6 +75,8 @@ public class Count extends FsCommand {
"The -" + OPTION_HUMAN +
" option shows file sizes in human readable format.\n" +
"The -" + OPTION_HEADER + " option displays a header line.\n" +
"The -" + OPTION_EXCLUDE_SNAPSHOT + " option excludes snapshots " +
"from being calculated. \n" +
"The -" + OPTION_TYPE + " option displays quota by storage types.\n" +
"It must be used with -" + OPTION_QUOTA + " option.\n" +
"If a comma-separated list of storage types is given after the -" +
@ -87,6 +92,7 @@ public class Count extends FsCommand {
private boolean showQuotabyType;
private List<StorageType> storageTypes = null;
private boolean showQuotasAndUsageOnly;
private boolean excludeSnapshots;
/** Constructor */
public Count() {}
@ -106,7 +112,8 @@ public class Count extends FsCommand {
@Override
protected void processOptions(LinkedList<String> args) {
CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE,
OPTION_QUOTA, OPTION_HUMAN, OPTION_HEADER, OPTION_QUOTA_AND_USAGE);
OPTION_QUOTA, OPTION_HUMAN, OPTION_HEADER, OPTION_QUOTA_AND_USAGE,
OPTION_EXCLUDE_SNAPSHOT);
cf.addOptionWithValue(OPTION_TYPE);
cf.parse(args);
if (args.isEmpty()) { // default path is the current working directory
@ -115,6 +122,7 @@ public class Count extends FsCommand {
showQuotas = cf.getOpt(OPTION_QUOTA);
humanReadable = cf.getOpt(OPTION_HUMAN);
showQuotasAndUsageOnly = cf.getOpt(OPTION_QUOTA_AND_USAGE);
excludeSnapshots = cf.getOpt(OPTION_EXCLUDE_SNAPSHOT);
if (showQuotas || showQuotasAndUsageOnly) {
String types = cf.getOptValue(OPTION_TYPE);
@ -125,6 +133,11 @@ public class Count extends FsCommand {
} else {
showQuotabyType = false;
}
if (excludeSnapshots) {
out.println(OPTION_QUOTA + " or " + OPTION_QUOTA_AND_USAGE + " option "
+ "is given, the -x option is ignored.");
excludeSnapshots = false;
}
}
if (cf.getOpt(OPTION_HEADER)) {
@ -163,7 +176,8 @@ public class Count extends FsCommand {
storageTypes) + src);
} else {
ContentSummary summary = src.fs.getContentSummary(src.path);
out.println(summary.toString(showQuotas, isHumanReadable()) + src);
out.println(summary.
toString(showQuotas, isHumanReadable(), excludeSnapshots) + src);
}
}

21
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsUsage.java
View File

@ -107,27 +107,30 @@ class FsUsage extends FsCommand {
/** show disk usage */
public static class Du extends FsUsage {
public static final String NAME = "du";
public static final String USAGE = "[-s] [-h] <path> ...";
public static final String USAGE = "[-s] [-h] [-x] <path> ...";
public static final String DESCRIPTION =
"Show the amount of space, in bytes, used by the files that " +
"match the specified file pattern. The following flags are optional:\n" +
"Show the amount of space, in bytes, used by the files that match " +
"the specified file pattern. The following flags are optional:\n" +
"-s: Rather than showing the size of each individual file that" +
" matches the pattern, shows the total (summary) size.\n" +
"-h: Formats the sizes of files in a human-readable fashion" +
" rather than a number of bytes.\n\n" +
"Note that, even without the -s option, this only shows size summaries " +
"one level deep into a directory.\n\n" +
" rather than a number of bytes.\n" +
"-x: Excludes snapshots from being counted.\n\n" +
"Note that, even without the -s option, this only shows size " +
"summaries one level deep into a directory.\n\n" +
"The output is in the form \n" +
"\tsize\tdisk space consumed\tname(full path)\n";
protected boolean summary = false;
private boolean excludeSnapshots = false;
@Override
protected void processOptions(LinkedList<String> args) throws IOException {
CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE, "h", "s");
CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE, "h", "s", "x");
cf.parse(args);
humanReadable = cf.getOpt("h");
summary = cf.getOpt("s");
excludeSnapshots = cf.getOpt("x");
if (args.isEmpty()) args.add(Path.CUR_DIR);
}
@ -156,6 +159,10 @@ class FsUsage extends FsCommand {
ContentSummary contentSummary = item.fs.getContentSummary(item.path);
long length = contentSummary.getLength();
long spaceConsumed = contentSummary.getSpaceConsumed();
if (excludeSnapshots) {
length -= contentSummary.getSnapshotLength();
spaceConsumed -= contentSummary.getSnapshotSpaceConsumed();
}
usagesTable.addRow(formatSize(length), formatSize(spaceConsumed), item);
}
}

11
hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
View File

@ -132,10 +132,12 @@ Similar to get command, except that the destination is restricted to a local fil
count
-----
Usage: `hadoop fs -count [-q] [-h] [-v] [-t [<storage type>]] [-u] <paths> `
Usage: `hadoop fs -count [-q] [-h] [-v] [-x] [-t [<storage type>]] [-u] <paths> `
Count the number of directories, files and bytes under the paths that match the specified file pattern. Get the quota and the usage. The output columns with -count are: DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME
The -u and -q options control what columns the output contains. -q means show quotas, -u limits the output to show quotas and usage only.
The output columns with -count -q are: QUOTA, REMAINING\_QUOTA, SPACE\_QUOTA, REMAINING\_SPACE\_QUOTA, DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME
The output columns with -count -u are: QUOTA, REMAINING\_QUOTA, SPACE\_QUOTA, REMAINING\_SPACE\_QUOTA
@ -146,6 +148,8 @@ The -h option shows sizes in human readable format.
The -v option displays a header line.
The -x option excludes snapshots from the result calculation. Without the -x option (default), the result is always calculated from all INodes, including all snapshots under the given path. The -x option is ignored if -u or -q option is given.
Example:
* `hadoop fs -count hdfs://nn1.example.com/file1 hdfs://nn2.example.com/file2`
@ -211,14 +215,15 @@ Example:
du
----
Usage: `hadoop fs -du [-s] [-h] URI [URI ...]`
Usage: `hadoop fs -du [-s] [-h] [-x] URI [URI ...]`
Displays sizes of files and directories contained in the given directory or the length of a file in case its just a file.
Options:
* The -s option will result in an aggregate summary of file lengths being displayed, rather than the individual files.
* The -s option will result in an aggregate summary of file lengths being displayed, rather than the individual files. Without the -s option, calculation is done by going 1-level deep from the given path.
* The -h option will format file sizes in a "human-readable" fashion (e.g 64.0m instead of 67108864)
* The -x option will exclude snapshots from the result calculation. Without the -x option (default), the result is always calculated from all INodes, including all snapshots under the given path.
The du returns three columns with the following format:

5
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java
View File

@ -447,7 +447,7 @@ public class TestCount {
Count count = new Count();
String actual = count.getUsage();
String expected =
"-count [-q] [-h] [-v] [-t [<storage type>]] [-u] <path> ...";
"-count [-q] [-h] [-v] [-t [<storage type>]] [-u] [-x] <path> ...";
assertEquals("Count.getUsage", expected, actual);
}
@ -465,6 +465,7 @@ public class TestCount {
+ " DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME\n"
+ "The -h option shows file sizes in human readable format.\n"
+ "The -v option displays a header line.\n"
+ "The -x option excludes snapshots from being calculated. \n"
+ "The -t option displays quota by storage types.\n"
+ "It must be used with -q option.\n"
+ "If a comma-separated list of storage types is given after the -t option, \n"
@ -521,7 +522,7 @@ public class TestCount {
}
@Override
public String toString(boolean qOption, boolean hOption) {
public String toString(boolean qOption, boolean hOption, boolean xOption) {
if (qOption) {
if (hOption) {
return (HUMAN + WITH_QUOTAS);

12
hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
View File

@ -200,7 +200,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
<expected-output>^-du \[-s\] \[-h\] &lt;path&gt; \.\.\. :\s*</expected-output>
<expected-output>^-du \[-s\] \[-h\] \[-x\] &lt;path&gt; \.\.\. :\s*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
@ -226,6 +226,10 @@
<type>RegexpComparator</type>
<expected-output>\s*of bytes.\s*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
<expected-output>^\s*-x\s*Excludes snapshots from being counted.\s*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
<expected-output>^\s*Note that, even without the -s option, this only shows size summaries one level\s*</expected-output>
@ -274,7 +278,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
<expected-output>^-count \[-q\] \[-h\] \[-v\] \[-t \[&lt;storage type&gt;\]\] \[-u\] &lt;path&gt; \.\.\. :( )*</expected-output>
<expected-output>^-count \[-q\] \[-h\] \[-v\] \[-t \[&lt;storage type&gt;\]\] \[-u\] \[-x\] &lt;path&gt; \.\.\. :( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
@ -308,6 +312,10 @@
<type>RegexpComparator</type>
<expected-output>^( |\t)*The -v option displays a header line.( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
<expected-output>^( |\t)*The -x option excludes snapshots from being calculated.( )*</expected-output>
</comparator>
</comparators>
</test>

8
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
View File

@ -1466,6 +1466,10 @@ public class PBHelperClient {
builder.length(cs.getLength()).
fileCount(cs.getFileCount()).
directoryCount(cs.getDirectoryCount()).
snapshotLength(cs.getSnapshotLength()).
snapshotFileCount(cs.getSnapshotFileCount()).
snapshotDirectoryCount(cs.getSnapshotDirectoryCount()).
snapshotSpaceConsumed(cs.getSnapshotSpaceConsumed()).
quota(cs.getQuota()).
spaceConsumed(cs.getSpaceConsumed()).
spaceQuota(cs.getSpaceQuota());
@ -2069,6 +2073,10 @@ public class PBHelperClient {
builder.setLength(cs.getLength()).
setFileCount(cs.getFileCount()).
setDirectoryCount(cs.getDirectoryCount()).
setSnapshotLength(cs.getSnapshotLength()).
setSnapshotFileCount(cs.getSnapshotFileCount()).
setSnapshotDirectoryCount(cs.getSnapshotDirectoryCount()).
setSnapshotSpaceConsumed(cs.getSnapshotSpaceConsumed()).
setQuota(cs.getQuota()).
setSpaceConsumed(cs.getSpaceConsumed()).
setSpaceQuota(cs.getSpaceQuota());

4
hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
View File

@ -138,6 +138,10 @@ message ContentSummaryProto {
required uint64 spaceConsumed = 5;
required uint64 spaceQuota = 6;
optional StorageTypeQuotaInfosProto typeQuotaInfos = 7;
optional uint64 snapshotLength = 8;
optional uint64 snapshotFileCount = 9;
optional uint64 snapshotDirectoryCount = 10;
optional uint64 snapshotSpaceConsumed = 11;
}
/**

6
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java
View File

@ -29,6 +29,7 @@ public class ContentSummaryComputationContext {
private FSNamesystem fsn = null;
private BlockStoragePolicySuite bsps = null;
private ContentCounts counts = null;
private ContentCounts snapshotCounts = null;
private long nextCountLimit = 0;
private long limitPerRun = 0;
private long yieldCount = 0;
@ -51,6 +52,7 @@ public class ContentSummaryComputationContext {
this.limitPerRun = limitPerRun;
this.nextCountLimit = limitPerRun;
this.counts = new ContentCounts.Builder().build();
this.snapshotCounts = new ContentCounts.Builder().build();
this.sleepMilliSec = sleepMicroSec/1000;
this.sleepNanoSec = (int)((sleepMicroSec%1000)*1000);
}
@ -125,6 +127,10 @@ public class ContentSummaryComputationContext {
return counts;
}
public ContentCounts getSnapshotCounts() {
return snapshotCounts;
}
public BlockStoragePolicySuite getBlockStoragePolicySuite() {
Preconditions.checkState((bsps != null || fsn != null),
"BlockStoragePolicySuite must be either initialized or available via" +

9
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
View File

@ -428,8 +428,9 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
*/
public final ContentSummary computeAndConvertContentSummary(int snapshotId,
ContentSummaryComputationContext summary) {
ContentCounts counts = computeContentSummary(snapshotId, summary)
.getCounts();
computeContentSummary(snapshotId, summary);
final ContentCounts counts = summary.getCounts();
final ContentCounts snapshotCounts = summary.getSnapshotCounts();
final QuotaCounts q = getQuotaCounts();
return new ContentSummary.Builder().
length(counts.getLength()).
@ -440,6 +441,10 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
spaceQuota(q.getStorageSpace()).
typeConsumed(counts.getTypeSpaces()).
typeQuota(q.getTypeSpaces().asArray()).
snapshotLength(snapshotCounts.getLength()).
snapshotFileCount(snapshotCounts.getFileCount()).
snapshotDirectoryCount(snapshotCounts.getDirectoryCount()).
snapshotSpaceConsumed(snapshotCounts.getStoragespace()).
build();
}

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
View File

@ -634,6 +634,10 @@ public class INodeDirectory extends INodeWithAdditionalFields
// computation should include all the deleted files/directories
sf.computeContentSummary4Snapshot(summary.getBlockStoragePolicySuite(),
summary.getCounts());
// Also compute ContentSummary for snapshotCounts (So we can extract it
// later from the ContentSummary of all).
sf.computeContentSummary4Snapshot(summary.getBlockStoragePolicySuite(),
summary.getSnapshotCounts());
}
final DirectoryWithQuotaFeature q = getDirectoryWithQuotaFeature();
if (q != null && snapshotId == Snapshot.CURRENT_STATE_ID) {

233
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
View File

@ -113,6 +113,11 @@ public class TestDFSShell {
return p;
}
static void rmr(FileSystem fs, Path p) throws IOException {
assertTrue(fs.delete(p, true));
assertFalse(fs.exists(p));
}
static File createLocalFile(File f) throws IOException {
assertTrue(!f.exists());
PrintWriter out = new PrintWriter(f);
@ -224,6 +229,7 @@ public class TestDFSShell {
shell.setConf(conf);
try {
cluster.waitActive();
Path myPath = new Path("/test/dir");
assertTrue(fs.mkdirs(myPath));
assertTrue(fs.exists(myPath));
@ -251,7 +257,7 @@ public class TestDFSShell {
assertTrue(val == 0);
String returnString = out.toString();
out.reset();
// Check if size matchs as expected
// Check if size matches as expected
assertThat(returnString, containsString(myFileLength.toString()));
assertThat(returnString, containsString(myFileDiskUsed.toString()));
assertThat(returnString, containsString(myFile2Length.toString()));
@ -308,7 +314,232 @@ public class TestDFSShell {
System.setOut(psBackup);
cluster.shutdown();
}
}
@Test (timeout = 180000)
public void testDuSnapshots() throws IOException {
final int replication = 2;
final Configuration conf = new HdfsConfiguration();
final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(replication).build();
final DistributedFileSystem dfs = cluster.getFileSystem();
final PrintStream psBackup = System.out;
final ByteArrayOutputStream out = new ByteArrayOutputStream();
final PrintStream psOut = new PrintStream(out);
final FsShell shell = new FsShell();
shell.setConf(conf);
try {
System.setOut(psOut);
cluster.waitActive();
final Path parent = new Path("/test");
final Path dir = new Path(parent, "dir");
mkdir(dfs, dir);
final Path file = new Path(dir, "file");
writeFile(dfs, file);
final Path file2 = new Path(dir, "file2");
writeFile(dfs, file2);
final Long fileLength = dfs.getFileStatus(file).getLen();
final Long fileDiskUsed = fileLength * replication;
final Long file2Length = dfs.getFileStatus(file2).getLen();
final Long file2DiskUsed = file2Length * replication;
/*
* Construct dir as follows:
* /test/dir/file <- this will later be deleted after snapshot taken.
* /test/dir/newfile <- this will be created after snapshot taken.
* /test/dir/file2
* Snapshot enabled on /test
*/
// test -du on /test/dir
int ret = -1;
try {
ret = shell.run(new String[] {"-du", dir.toString()});
} catch (Exception e) {
System.err.println("Exception raised from DFSShell.run " +
e.getLocalizedMessage());
}
assertEquals(0, ret);
String returnString = out.toString();
LOG.info("-du return is:\n" + returnString);
// Check if size matches as expected
assertTrue(returnString.contains(fileLength.toString()));
assertTrue(returnString.contains(fileDiskUsed.toString()));
assertTrue(returnString.contains(file2Length.toString()));
assertTrue(returnString.contains(file2DiskUsed.toString()));
out.reset();
// take a snapshot, then remove file and add newFile
final String snapshotName = "ss1";
final Path snapshotPath = new Path(parent, ".snapshot/" + snapshotName);
dfs.allowSnapshot(parent);
assertThat(dfs.createSnapshot(parent, snapshotName), is(snapshotPath));
rmr(dfs, file);
final Path newFile = new Path(dir, "newfile");
writeFile(dfs, newFile);
final Long newFileLength = dfs.getFileStatus(newFile).getLen();
final Long newFileDiskUsed = newFileLength * replication;
// test -du -s on /test
ret = -1;
try {
ret = shell.run(new String[] {"-du", "-s", parent.toString()});
} catch (Exception e) {
System.err.println("Exception raised from DFSShell.run " +
e.getLocalizedMessage());
}
assertEquals(0, ret);
returnString = out.toString();
LOG.info("-du -s return is:\n" + returnString);
Long combinedLength = fileLength + file2Length + newFileLength;
Long combinedDiskUsed = fileDiskUsed + file2DiskUsed + newFileDiskUsed;
assertTrue(returnString.contains(combinedLength.toString()));
assertTrue(returnString.contains(combinedDiskUsed.toString()));
out.reset();
// test -du on /test
ret = -1;
try {
ret = shell.run(new String[] {"-du", parent.toString()});
} catch (Exception e) {
System.err.println("Exception raised from DFSShell.run " +
e.getLocalizedMessage());
}
assertEquals(0, ret);
returnString = out.toString();
LOG.info("-du return is:\n" + returnString);
assertTrue(returnString.contains(combinedLength.toString()));
assertTrue(returnString.contains(combinedDiskUsed.toString()));
out.reset();
// test -du -s -x on /test
ret = -1;
try {
ret = shell.run(new String[] {"-du", "-s", "-x", parent.toString()});
} catch (Exception e) {
System.err.println("Exception raised from DFSShell.run " +
e.getLocalizedMessage());
}
assertEquals(0, ret);
returnString = out.toString();
LOG.info("-du -s -x return is:\n" + returnString);
Long exludeSnapshotLength = file2Length + newFileLength;
Long excludeSnapshotDiskUsed = file2DiskUsed + newFileDiskUsed;
assertTrue(returnString.contains(exludeSnapshotLength.toString()));
assertTrue(returnString.contains(excludeSnapshotDiskUsed.toString()));
out.reset();
// test -du -x on /test
ret = -1;
try {
ret = shell.run(new String[] {"-du", "-x", parent.toString()});
} catch (Exception e) {
System.err.println("Exception raised from DFSShell.run " +
e.getLocalizedMessage());
}
assertEquals(0, ret);
returnString = out.toString();
LOG.info("-du -x return is:\n" + returnString);
assertTrue(returnString.contains(exludeSnapshotLength.toString()));
assertTrue(returnString.contains(excludeSnapshotDiskUsed.toString()));
out.reset();
} finally {
System.setOut(psBackup);
cluster.shutdown();
}
}
@Test (timeout = 180000)
public void testCountSnapshots() throws IOException {
final int replication = 2;
final Configuration conf = new HdfsConfiguration();
final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(replication).build();
final DistributedFileSystem dfs = cluster.getFileSystem();
final PrintStream psBackup = System.out;
final ByteArrayOutputStream out = new ByteArrayOutputStream();
final PrintStream psOut = new PrintStream(out);
System.setOut(psOut);
final FsShell shell = new FsShell();
shell.setConf(conf);
try {
cluster.waitActive();
final Path parent = new Path("/test");
final Path dir = new Path(parent, "dir");
mkdir(dfs, dir);
final Path file = new Path(dir, "file");
writeFile(dfs, file);
final Path file2 = new Path(dir, "file2");
writeFile(dfs, file2);
final long fileLength = dfs.getFileStatus(file).getLen();
final long file2Length = dfs.getFileStatus(file2).getLen();
final Path dir2 = new Path(parent, "dir2");
mkdir(dfs, dir2);
/*
* Construct dir as follows:
* /test/dir/file <- this will later be deleted after snapshot taken.
* /test/dir/newfile <- this will be created after snapshot taken.
* /test/dir/file2
* /test/dir2 <- this will later be deleted after snapshot taken.
* Snapshot enabled on /test
*/
// take a snapshot
// then create /test/dir/newfile and remove /test/dir/file, /test/dir2
final String snapshotName = "s1";
final Path snapshotPath = new Path(parent, ".snapshot/" + snapshotName);
dfs.allowSnapshot(parent);
assertThat(dfs.createSnapshot(parent, snapshotName), is(snapshotPath));
rmr(dfs, file);
rmr(dfs, dir2);
final Path newFile = new Path(dir, "new file");
writeFile(dfs, newFile);
final Long newFileLength = dfs.getFileStatus(newFile).getLen();
// test -count on /test. Include header for easier debugging.
int val = -1;
try {
val = shell.run(new String[] {"-count", "-v", parent.toString() });
} catch (Exception e) {
System.err.println("Exception raised from DFSShell.run " +
e.getLocalizedMessage());
}
assertEquals(0, val);
String returnString = out.toString();
LOG.info("-count return is:\n" + returnString);
Scanner in = new Scanner(returnString);
in.nextLine();
assertEquals(3, in.nextLong()); //DIR_COUNT
assertEquals(3, in.nextLong()); //FILE_COUNT
assertEquals(fileLength + file2Length + newFileLength,
in.nextLong()); //CONTENT_SIZE
out.reset();
// test -count -x on /test. Include header for easier debugging.
val = -1;
try {
val =
shell.run(new String[] {"-count", "-x", "-v", parent.toString()});
} catch (Exception e) {
System.err.println("Exception raised from DFSShell.run " +
e.getLocalizedMessage());
}
assertEquals(0, val);
returnString = out.toString();
LOG.info("-count -x return is:\n" + returnString);
in = new Scanner(returnString);
in.nextLine();
assertEquals(2, in.nextLong()); //DIR_COUNT
assertEquals(2, in.nextLong()); //FILE_COUNT
assertEquals(file2Length + newFileLength, in.nextLong()); //CONTENT_SIZE
out.reset();
} finally {
System.setOut(psBackup);
cluster.shutdown();
}
}
@Test (timeout = 30000)