Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-16954. Add -S option in "Count" command to show only Snapshot Counts. Contributed by hemanthboyina. 

(cherry picked from commit b89d875f7b)
This commit is contained in:
Takanobu Asanuma 2020-04-06 11:03:10 +09:00 committed by S O'Donnell
parent e944cc0338
commit 65bf544118
6 changed files with 110 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
View File

@ -281,6 +281,21 @@ public class ContentSummary extends QuotaUsage implements Writable{
private static final String ALL_HEADER = QUOTA_HEADER + SUMMARY_HEADER;
/**
* Output format:<-------18-------> <----------24---------->
* <----------24---------->. <-------------28------------> SNAPSHOT_LENGTH
* SNAPSHOT_FILE_COUNT SNAPSHOT_DIR_COUNT SNAPSHOT_SPACE_CONSUMED
*/
private static final String SNAPSHOT_FORMAT = "%18s %24s %24s %28s ";
private static final String[] SNAPSHOT_HEADER_FIELDS =
new String[] {"SNAPSHOT_LENGTH", "SNAPSHOT_FILE_COUNT",
"SNAPSHOT_DIR_COUNT", "SNAPSHOT_SPACE_CONSUMED"};
/** The header string. */
private static final String SNAPSHOT_HEADER =
String.format(SNAPSHOT_FORMAT, (Object[]) SNAPSHOT_HEADER_FIELDS);
/** Return the header of the output.
* if qOption is false, output directory count, file count, and content size;
@ -293,7 +308,9 @@ public class ContentSummary extends QuotaUsage implements Writable{
return qOption ? ALL_HEADER : SUMMARY_HEADER;
}
public static String getSnapshotHeader() {
return SNAPSHOT_HEADER;
}
/**
* Returns the names of the fields from the summary header.
@ -416,7 +433,7 @@ public class ContentSummary extends QuotaUsage implements Writable{
}
/**
* Formats a size to be human readable or in bytes
* Formats a size to be human readable or in bytes.
* @param size value to be formatted
* @param humanReadable flag indicating human readable or not
* @return String representation of the size
@ -426,4 +443,17 @@ public class ContentSummary extends QuotaUsage implements Writable{
? StringUtils.TraditionalBinaryPrefix.long2String(size, "", 1)
: String.valueOf(size);
}
/**
* Return the string representation of the snapshot counts in the output
* format.
* @param hOption flag indicating human readable or not
* @return String representation of the snapshot counts
*/
public String toSnapshot(boolean hOption) {
return String.format(SNAPSHOT_FORMAT, formatSize(snapshotLength, hOption),
formatSize(snapshotFileCount, hOption),
formatSize(snapshotDirectoryCount, hOption),
formatSize(snapshotSpaceConsumed, hOption));
}
}

17
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
View File

@ -56,13 +56,14 @@ public class Count extends FsCommand {
//return the quota, namespace count and disk space usage.
private static final String OPTION_QUOTA_AND_USAGE = "u";
private static final String OPTION_ECPOLICY = "e";
private static final String OPTION_SNAPSHOT_COUNT = "s";
public static final String NAME = "count";
public static final String USAGE =
"[-" + OPTION_QUOTA + "] [-" + OPTION_HUMAN + "] [-" + OPTION_HEADER
+ "] [-" + OPTION_TYPE + " [<storage type>]] [-" +
OPTION_QUOTA_AND_USAGE + "] [-" + OPTION_EXCLUDE_SNAPSHOT
+ "] [-" + OPTION_ECPOLICY
+ "] [-" + OPTION_ECPOLICY + "] [-" + OPTION_SNAPSHOT_COUNT
+ "] <path> ...";
public static final String DESCRIPTION =
"Count the number of directories, files and bytes under the paths\n" +
@ -93,7 +94,8 @@ public class Count extends FsCommand {
"the storage types.\n" +
"The -" + OPTION_QUOTA_AND_USAGE + " option shows the quota and \n" +
"the usage against the quota without the detailed content summary."+
"The -"+ OPTION_ECPOLICY +" option shows the erasure coding policy.";
"The -" + OPTION_ECPOLICY + " option shows the erasure coding policy."
+ "The -" + OPTION_SNAPSHOT_COUNT + " option shows snapshot counts.";
private boolean showQuotas;
private boolean humanReadable;
@ -102,6 +104,7 @@ public class Count extends FsCommand {
private boolean showQuotasAndUsageOnly;
private boolean excludeSnapshots;
private boolean displayECPolicy;
private boolean showSnapshot;
/** Constructor */
public Count() {}
@ -123,7 +126,7 @@ public class Count extends FsCommand {
CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE,
OPTION_QUOTA, OPTION_HUMAN, OPTION_HEADER, OPTION_QUOTA_AND_USAGE,
OPTION_EXCLUDE_SNAPSHOT,
OPTION_ECPOLICY);
OPTION_ECPOLICY, OPTION_SNAPSHOT_COUNT);
cf.addOptionWithValue(OPTION_TYPE);
cf.parse(args);
if (args.isEmpty()) { // default path is the current working directory
@ -134,6 +137,7 @@ public class Count extends FsCommand {
showQuotasAndUsageOnly = cf.getOpt(OPTION_QUOTA_AND_USAGE);
excludeSnapshots = cf.getOpt(OPTION_EXCLUDE_SNAPSHOT);
displayECPolicy = cf.getOpt(OPTION_ECPOLICY);
showSnapshot = cf.getOpt(OPTION_SNAPSHOT_COUNT);
if (showQuotas || showQuotasAndUsageOnly) {
String types = cf.getOptValue(OPTION_TYPE);
@ -165,6 +169,9 @@ public class Count extends FsCommand {
if(displayECPolicy){
headString.append("ERASURECODING_POLICY ");
}
if (showSnapshot) {
headString.append(ContentSummary.getSnapshotHeader());
}
headString.append("PATHNAME");
out.println(headString.toString());
}
@ -205,6 +212,10 @@ public class Count extends FsCommand {
outputString.append(summary.getErasureCodingPolicy())
.append(" ");
}
if (showSnapshot) {
ContentSummary summary = src.fs.getContentSummary(src.path);
outputString.append(summary.toSnapshot(isHumanReadable()));
}
outputString.append(src);
out.println(outputString.toString());
}

5
hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
View File

@ -132,7 +132,7 @@ Identical to the -get command.
count
-----
Usage: `hadoop fs -count [-q] [-h] [-v] [-x] [-t [<storage type>]] [-u] [-e] <paths> `
Usage: `hadoop fs -count [-q] [-h] [-v] [-x] [-t [<storage type>]] [-u] [-e] [-s] <paths> `
Count the number of directories, files and bytes under the paths that match the specified file pattern. Get the quota and the usage. The output columns with -count are: DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME
@ -156,6 +156,8 @@ The output columns with -count -e are: DIR\_COUNT, FILE\_COUNT, CONTENT_SIZE, ER
The ERASURECODING\_POLICY is name of the policy for the file. If a erasure coding policy is setted on that file, it will return name of the policy. If no erasure coding policy is setted, it will return \"Replicated\" which means it use replication storage strategy.
The -s option shows the snapshot counts for each directory.
Example:
* `hadoop fs -count hdfs://nn1.example.com/file1 hdfs://nn2.example.com/file2`
@ -166,6 +168,7 @@ Example:
* `hadoop fs -count -u -h hdfs://nn1.example.com/file1`
* `hadoop fs -count -u -h -v hdfs://nn1.example.com/file1`
* `hadoop fs -count -e hdfs://nn1.example.com/file1`
* `hadoop fs -count -s hdfs://nn1.example.com/file1`
Exit Code:

36
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestContentSummary.java
View File

@ -253,4 +253,40 @@ public class TestContentSummary {
String expected = " 32.6 K 211.9 M 8.0 E ";
assertEquals(expected, contentSummary.toString(false, true));
}
// check the toSnapshot method with human readable.
@Test
public void testToSnapshotHumanReadable() {
long snapshotLength = Long.MAX_VALUE;
long snapshotFileCount = 222222222;
long snapshotDirectoryCount = 33333;
long snapshotSpaceConsumed = 222256578;
ContentSummary contentSummary = new ContentSummary.Builder()
.snapshotLength(snapshotLength).snapshotFileCount(snapshotFileCount)
.snapshotDirectoryCount(snapshotDirectoryCount)
.snapshotSpaceConsumed(snapshotSpaceConsumed).build();
String expected =
" 8.0 E 211.9 M 32.6 K "
+ " 212.0 M ";
assertEquals(expected, contentSummary.toSnapshot(true));
}
// check the toSnapshot method with human readable disabled.
@Test
public void testToSnapshotNotHumanReadable() {
long snapshotLength = 1111;
long snapshotFileCount = 2222;
long snapshotDirectoryCount = 3333;
long snapshotSpaceConsumed = 4444;
ContentSummary contentSummary = new ContentSummary.Builder()
.snapshotLength(snapshotLength).snapshotFileCount(snapshotFileCount)
.snapshotDirectoryCount(snapshotDirectoryCount)
.snapshotSpaceConsumed(snapshotSpaceConsumed).build();
String expected =
" 1111 2222 3333 "
+ " 4444 ";
assertEquals(expected, contentSummary.toSnapshot(false));
}
}

25
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java
View File

@ -411,6 +411,25 @@ public class TestCount {
verifyNoMoreInteractions(out);
}
@Test
public void processPathWithSnapshotHeader() throws Exception {
Path path = new Path("mockfs:/test");
when(mockFs.getFileStatus(eq(path))).thenReturn(fileStat);
PrintStream out = mock(PrintStream.class);
Count count = new Count();
count.out = out;
LinkedList<String> options = new LinkedList<String>();
options.add("-s");
options.add("-v");
options.add("dummy");
count.processOptions(options);
String withSnapshotHeader = " DIR_COUNT FILE_COUNT CONTENT_SIZE "
+ " SNAPSHOT_LENGTH SNAPSHOT_FILE_COUNT "
+ " SNAPSHOT_DIR_COUNT SNAPSHOT_SPACE_CONSUMED PATHNAME";
verify(out).println(withSnapshotHeader);
verifyNoMoreInteractions(out);
}
@Test
public void getCommandName() {
Count count = new Count();
@ -448,7 +467,8 @@ public class TestCount {
Count count = new Count();
String actual = count.getUsage();
String expected =
"-count [-q] [-h] [-v] [-t [<storage type>]] [-u] [-x] [-e] <path> ...";
"-count [-q] [-h] [-v] [-t [<storage type>]]"
+ " [-u] [-x] [-e] [-s] <path> ...";
assertEquals("Count.getUsage", expected, actual);
}
@ -480,7 +500,8 @@ public class TestCount {
+ "storage types.\n"
+ "The -u option shows the quota and \n"
+ "the usage against the quota without the detailed content summary."
+ "The -e option shows the erasure coding policy.";
+ "The -e option shows the erasure coding policy."
+ "The -s option shows snapshot counts.";
assertEquals("Count.getDescription", expected, actual);
}

2
hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
View File

@ -282,7 +282,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
<expected-output>^-count \[-q\] \[-h\] \[-v\] \[-t \[&lt;storage type&gt;\]\] \[-u\] \[-x\] \[-e\] &lt;path&gt; \.\.\. :( )*</expected-output>
<expected-output>^-count \[-q\] \[-h\] \[-v\] \[-t \[&lt;storage type&gt;\]\] \[-u\] \[-x\] \[-e\] \[-s\] &lt;path&gt; \.\.\. :( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>