Revert HADOOP-7202 since HDFS cannot be compiled.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1086455 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
50b1f9fc73
commit
bb4f277407
|
@ -84,8 +84,6 @@ Trunk (unreleased changes)
|
||||||
HADOOP-7180. Better support on CommandFormat on the API and exceptions.
|
HADOOP-7180. Better support on CommandFormat on the API and exceptions.
|
||||||
(Daryn Sharp via szetszwo)
|
(Daryn Sharp via szetszwo)
|
||||||
|
|
||||||
HADOOP-7202. Improve shell Command base class. (Daryn Sharp via szetszwo)
|
|
||||||
|
|
||||||
OPTIMIZATIONS
|
OPTIMIZATIONS
|
||||||
|
|
||||||
HADOOP-6949. Reduce RPC packet size of primitive arrays using
|
HADOOP-6949. Reduce RPC packet size of primitive arrays using
|
||||||
|
|
|
@ -370,7 +370,7 @@ public class FsShell extends Configured implements Tool {
|
||||||
|
|
||||||
private class TextRecordInputStream extends InputStream {
|
private class TextRecordInputStream extends InputStream {
|
||||||
SequenceFile.Reader r;
|
SequenceFile.Reader r;
|
||||||
WritableComparable<?> key;
|
WritableComparable key;
|
||||||
Writable val;
|
Writable val;
|
||||||
|
|
||||||
DataInputBuffer inbuf;
|
DataInputBuffer inbuf;
|
||||||
|
@ -1691,6 +1691,8 @@ public class FsShell extends Configured implements Tool {
|
||||||
delete(argv[i], true, rmSkipTrash);
|
delete(argv[i], true, rmSkipTrash);
|
||||||
} else if ("-df".equals(cmd)) {
|
} else if ("-df".equals(cmd)) {
|
||||||
df(argv[i]);
|
df(argv[i]);
|
||||||
|
} else if (Count.matches(cmd)) {
|
||||||
|
new Count(argv, i, getConf()).runAll();
|
||||||
} else if ("-ls".equals(cmd)) {
|
} else if ("-ls".equals(cmd)) {
|
||||||
exitCode = ls(argv[i], false);
|
exitCode = ls(argv[i], false);
|
||||||
} else if ("-lsr".equals(cmd)) {
|
} else if ("-lsr".equals(cmd)) {
|
||||||
|
@ -1757,7 +1759,7 @@ public class FsShell extends Configured implements Tool {
|
||||||
} else if ("-df".equals(cmd) ) {
|
} else if ("-df".equals(cmd) ) {
|
||||||
System.err.println("Usage: java FsShell" +
|
System.err.println("Usage: java FsShell" +
|
||||||
" [" + cmd + " [<path>]]");
|
" [" + cmd + " [<path>]]");
|
||||||
} else if ("-count".equals(cmd)) {
|
} else if (Count.matches(cmd)) {
|
||||||
System.err.println(prefix + " [" + Count.USAGE + "]");
|
System.err.println(prefix + " [" + Count.USAGE + "]");
|
||||||
} else if ("-rm".equals(cmd) || "-rmr".equals(cmd)) {
|
} else if ("-rm".equals(cmd) || "-rmr".equals(cmd)) {
|
||||||
System.err.println("Usage: java FsShell [" + cmd +
|
System.err.println("Usage: java FsShell [" + cmd +
|
||||||
|
@ -1946,13 +1948,8 @@ public class FsShell extends Configured implements Tool {
|
||||||
du(argv, i);
|
du(argv, i);
|
||||||
} else if ("-dus".equals(cmd)) {
|
} else if ("-dus".equals(cmd)) {
|
||||||
dus(argv, i);
|
dus(argv, i);
|
||||||
} else if ("-count".equals(cmd)) {
|
} else if (Count.matches(cmd)) {
|
||||||
// TODO: next two lines are a temporary crutch until this entire
|
exitCode = new Count(argv, i, getConf()).runAll();
|
||||||
// block is overhauled
|
|
||||||
LinkedList<String> args = new LinkedList<String>(Arrays.asList(argv));
|
|
||||||
args.removeFirst();
|
|
||||||
Count runner = ReflectionUtils.newInstance(Count.class, getConf());
|
|
||||||
exitCode = runner.run(cmd, args);
|
|
||||||
} else if ("-mkdir".equals(cmd)) {
|
} else if ("-mkdir".equals(cmd)) {
|
||||||
exitCode = doall(cmd, argv, i);
|
exitCode = doall(cmd, argv, i);
|
||||||
} else if ("-touchz".equals(cmd)) {
|
} else if ("-touchz".equals(cmd)) {
|
||||||
|
|
|
@ -17,53 +17,27 @@
|
||||||
*/
|
*/
|
||||||
package org.apache.hadoop.fs.shell;
|
package org.apache.hadoop.fs.shell;
|
||||||
|
|
||||||
import java.io.FileNotFoundException;
|
import java.io.*;
|
||||||
import java.io.IOException;
|
|
||||||
import java.io.PrintStream;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.LinkedList;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
import org.apache.commons.logging.Log;
|
|
||||||
import org.apache.commons.logging.LogFactory;
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
|
||||||
import org.apache.hadoop.classification.InterfaceStability;
|
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.conf.Configured;
|
import org.apache.hadoop.conf.Configured;
|
||||||
import org.apache.hadoop.fs.FileStatus;
|
import org.apache.hadoop.fs.FileStatus;
|
||||||
import org.apache.hadoop.fs.FileSystem;
|
import org.apache.hadoop.fs.FileSystem;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.util.StringUtils;
|
import org.apache.hadoop.ipc.RemoteException;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* An abstract class for the execution of a file system command
|
* An abstract class for the execution of a file system command
|
||||||
*/
|
*/
|
||||||
@InterfaceAudience.Private
|
|
||||||
@InterfaceStability.Evolving
|
|
||||||
|
|
||||||
abstract public class Command extends Configured {
|
abstract public class Command extends Configured {
|
||||||
protected String[] args;
|
protected String[] args;
|
||||||
protected String cmdName;
|
|
||||||
protected int exitCode = 0;
|
|
||||||
protected int numErrors = 0;
|
|
||||||
protected boolean recursive = false;
|
|
||||||
protected ArrayList<Exception> exceptions = new ArrayList<Exception>();
|
|
||||||
|
|
||||||
private static final Log LOG = LogFactory.getLog(Command.class);
|
|
||||||
|
|
||||||
/** allows stdout to be captured if necessary */
|
|
||||||
public PrintStream out = System.out;
|
|
||||||
/** allows stderr to be captured if necessary */
|
|
||||||
public PrintStream err = System.err;
|
|
||||||
|
|
||||||
protected Command() {}
|
|
||||||
|
|
||||||
/** Constructor */
|
/** Constructor */
|
||||||
protected Command(Configuration conf) {
|
protected Command(Configuration conf) {
|
||||||
super(conf);
|
super(conf);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @return the command's name excluding the leading character - */
|
/** Return the command's name excluding the leading character - */
|
||||||
abstract public String getCommandName();
|
abstract public String getCommandName();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -83,273 +57,30 @@ abstract public class Command extends Configured {
|
||||||
int exitCode = 0;
|
int exitCode = 0;
|
||||||
for (String src : args) {
|
for (String src : args) {
|
||||||
try {
|
try {
|
||||||
List<PathData> srcs = expandGlob(src);
|
Path srcPath = new Path(src);
|
||||||
for(PathData s : srcs) {
|
FileSystem fs = srcPath.getFileSystem(getConf());
|
||||||
run(s.path);
|
FileStatus[] statuses = fs.globStatus(srcPath);
|
||||||
|
if (statuses == null) {
|
||||||
|
System.err.println("Can not find listing for " + src);
|
||||||
|
exitCode = -1;
|
||||||
|
} else {
|
||||||
|
for(FileStatus s : statuses) {
|
||||||
|
run(s.getPath());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
} catch (RemoteException re) {
|
||||||
|
exitCode = -1;
|
||||||
|
String content = re.getLocalizedMessage();
|
||||||
|
int eol = content.indexOf('\n');
|
||||||
|
if (eol>=0) {
|
||||||
|
content = content.substring(0, eol);
|
||||||
|
}
|
||||||
|
System.err.println(getCommandName() + ": " + content);
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
exitCode = -1;
|
exitCode = -1;
|
||||||
displayError(e);
|
System.err.println(getCommandName() + ": " + e.getLocalizedMessage());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return exitCode;
|
return exitCode;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Invokes the command handler. The default behavior is to process options,
|
|
||||||
* expand arguments, and then process each argument.
|
|
||||||
* <pre>
|
|
||||||
* run
|
|
||||||
* \-> {@link #processOptions(LinkedList)}
|
|
||||||
* \-> {@link #expandArguments(LinkedList)} -> {@link #expandArgument(String)}*
|
|
||||||
* \-> {@link #processArguments(LinkedList)}
|
|
||||||
* \-> {@link #processArgument(PathData)}*
|
|
||||||
* \-> {@link #processPathArgument(PathData)}
|
|
||||||
* \-> {@link #processPaths(PathData, PathData...)}
|
|
||||||
* \-> {@link #processPath(PathData)}*
|
|
||||||
* \-> {@link #processNonexistentPathArgument(PathData)}
|
|
||||||
* </pre>
|
|
||||||
* Most commands will chose to implement just
|
|
||||||
* {@link #processOptions(LinkedList)} and {@link #processPath(PathData)}
|
|
||||||
*
|
|
||||||
* @param cmd the command's name
|
|
||||||
* @param args the list of command line arguments
|
|
||||||
* @return the exit code for the command
|
|
||||||
* @throws IllegalArgumentException if called with invalid arguments
|
|
||||||
*/
|
|
||||||
public int run(String cmd, LinkedList<String> args) {
|
|
||||||
cmdName = cmd;
|
|
||||||
try {
|
|
||||||
processOptions(args);
|
|
||||||
processArguments(expandArguments(args));
|
|
||||||
} catch (IOException e) {
|
|
||||||
displayError(e);
|
|
||||||
}
|
|
||||||
|
|
||||||
return (numErrors == 0) ? exitCode : 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Must be implemented by commands to process the command line flags and
|
|
||||||
* check the bounds of the remaining arguments. If an
|
|
||||||
* IllegalArgumentException is thrown, the FsShell object will print the
|
|
||||||
* short usage of the command.
|
|
||||||
* @param args the command line arguments
|
|
||||||
* @throws IOException
|
|
||||||
*/
|
|
||||||
protected void processOptions(LinkedList<String> args) throws IOException {}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Expands a list of arguments into {@link PathData} objects. The default
|
|
||||||
* behavior is to call {@link #expandArgument(String)} on each element
|
|
||||||
* which by default globs the argument. The loop catches IOExceptions,
|
|
||||||
* increments the error count, and displays the exception.
|
|
||||||
* @param args strings to expand into {@link PathData} objects
|
|
||||||
* @return list of all {@link PathData} objects the arguments
|
|
||||||
* @throws IOException if anything goes wrong...
|
|
||||||
*/
|
|
||||||
protected LinkedList<PathData> expandArguments(LinkedList<String> args)
|
|
||||||
throws IOException {
|
|
||||||
LinkedList<PathData> expandedArgs = new LinkedList<PathData>();
|
|
||||||
for (String arg : args) {
|
|
||||||
try {
|
|
||||||
expandedArgs.addAll(expandArgument(arg));
|
|
||||||
} catch (IOException e) { // other exceptions are probably nasty
|
|
||||||
displayError(e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return expandedArgs;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Expand the given argument into a list of {@link PathData} objects.
|
|
||||||
* The default behavior is to expand globs. Commands may override to
|
|
||||||
* perform other expansions on an argument.
|
|
||||||
* @param arg string pattern to expand
|
|
||||||
* @return list of {@link PathData} objects
|
|
||||||
* @throws IOException if anything goes wrong...
|
|
||||||
*/
|
|
||||||
protected List<PathData> expandArgument(String arg) throws IOException {
|
|
||||||
return expandGlob(arg);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Processes the command's list of expanded arguments.
|
|
||||||
* {@link #processArgument(PathData)} will be invoked with each item
|
|
||||||
* in the list. The loop catches IOExceptions, increments the error
|
|
||||||
* count, and displays the exception.
|
|
||||||
* @param args a list of {@link PathData} to process
|
|
||||||
* @throws IOException if anything goes wrong...
|
|
||||||
*/
|
|
||||||
protected void processArguments(LinkedList<PathData> args)
|
|
||||||
throws IOException {
|
|
||||||
for (PathData arg : args) {
|
|
||||||
try {
|
|
||||||
processArgument(arg);
|
|
||||||
} catch (IOException e) {
|
|
||||||
displayError(e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Processes a {@link PathData} item, calling
|
|
||||||
* {@link #processPathArgument(PathData)} or
|
|
||||||
* {@link #processNonexistentPathArgument(PathData)} on each item.
|
|
||||||
* @param arg {@link PathData} item to process
|
|
||||||
* @throws IOException if anything goes wrong...
|
|
||||||
*/
|
|
||||||
protected void processArgument(PathData item) throws IOException {
|
|
||||||
if (item.exists) {
|
|
||||||
processPathArgument(item);
|
|
||||||
} else {
|
|
||||||
processNonexistentPathArgument(item);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* This is the last chance to modify an argument before going into the
|
|
||||||
* (possibly) recursive {@link #processPaths(PathData, PathData...)}
|
|
||||||
* -> {@link #processPath(PathData)} loop. Ex. ls and du use this to
|
|
||||||
* expand out directories.
|
|
||||||
* @param item a {@link PathData} representing a path which exists
|
|
||||||
* @throws IOException if anything goes wrong...
|
|
||||||
*/
|
|
||||||
protected void processPathArgument(PathData item) throws IOException {
|
|
||||||
// null indicates that the call is not via recursion, ie. there is
|
|
||||||
// no parent directory that was expanded
|
|
||||||
processPaths(null, item);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Provides a hook for handling paths that don't exist. By default it
|
|
||||||
* will throw an exception. Primarily overriden by commands that create
|
|
||||||
* paths such as mkdir or touch.
|
|
||||||
* @param item the {@link PathData} that doesn't exist
|
|
||||||
* @throws FileNotFoundException if arg is a path and it doesn't exist
|
|
||||||
* @throws IOException if anything else goes wrong...
|
|
||||||
*/
|
|
||||||
protected void processNonexistentPathArgument(PathData item)
|
|
||||||
throws IOException {
|
|
||||||
throw new FileNotFoundException(item + ": No such file or directory");
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Iterates over the given expanded paths and invokes
|
|
||||||
* {@link #processPath(PathData)} on each element. If "recursive" is true,
|
|
||||||
* will do a post-visit DFS on directories.
|
|
||||||
* @param parent if called via a recurse, will be the parent dir, else null
|
|
||||||
* @param items a list of {@link PathData} objects to process
|
|
||||||
* @throws IOException if anything goes wrong...
|
|
||||||
*/
|
|
||||||
protected void processPaths(PathData parent, PathData ... items)
|
|
||||||
throws IOException {
|
|
||||||
// TODO: this really should be iterative
|
|
||||||
for (PathData item : items) {
|
|
||||||
try {
|
|
||||||
processPath(item);
|
|
||||||
if (recursive && item.stat.isDirectory()) {
|
|
||||||
recursePath(item);
|
|
||||||
}
|
|
||||||
} catch (IOException e) {
|
|
||||||
displayError(e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Hook for commands to implement an operation to be applied on each
|
|
||||||
* path for the command. Note implementation of this method is optional
|
|
||||||
* if earlier methods in the chain handle the operation.
|
|
||||||
* @param item a {@link PathData} object
|
|
||||||
* @throws RuntimeException if invoked but not implemented
|
|
||||||
* @throws IOException if anything else goes wrong in the user-implementation
|
|
||||||
*/
|
|
||||||
protected void processPath(PathData item) throws IOException {
|
|
||||||
throw new RuntimeException("processPath() is not implemented");
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Gets the directory listing for a path and invokes
|
|
||||||
* {@link #processPaths(PathData, PathData...)}
|
|
||||||
* @param item {@link PathData} for directory to recurse into
|
|
||||||
* @throws IOException if anything goes wrong...
|
|
||||||
*/
|
|
||||||
protected void recursePath(PathData item) throws IOException {
|
|
||||||
processPaths(item, item.getDirectoryContents());
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Expand the given path as a glob pattern. Non-existent paths do not
|
|
||||||
* throw an exception because creation commands like touch and mkdir need
|
|
||||||
* to create them. The "stat" field will be null if the path does not
|
|
||||||
* exist.
|
|
||||||
* @param pattern the glob to expand
|
|
||||||
* @return list of {@link PathData} objects
|
|
||||||
* @throws FileNotFoundException the path is a glob with no matches
|
|
||||||
* @throws IOException anything else goes wrong...
|
|
||||||
*/
|
|
||||||
protected List<PathData> expandGlob(String pattern) throws IOException {
|
|
||||||
Path path = new Path(pattern);
|
|
||||||
FileSystem fs = path.getFileSystem(getConf());
|
|
||||||
FileStatus[] stats = fs.globStatus(path);
|
|
||||||
|
|
||||||
if (stats != null && stats.length == 0) { // glob failed to match
|
|
||||||
throw new FileNotFoundException(path + ": No such file or directory");
|
|
||||||
}
|
|
||||||
|
|
||||||
List<PathData> items = new LinkedList<PathData>();
|
|
||||||
if (stats == null) { // not a glob & file not found, so null stat block
|
|
||||||
items.add(new PathData(fs, path, null));
|
|
||||||
} else {
|
|
||||||
// convert all the stats to PathData objs
|
|
||||||
for (FileStatus stat : stats) {
|
|
||||||
items.add(new PathData(fs, stat));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return items;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Display an exception prefaced with the command name. Also increments
|
|
||||||
* the error count for the command which will result in a non-zero exit
|
|
||||||
* code.
|
|
||||||
* @param e exception to display
|
|
||||||
*/
|
|
||||||
public void displayError(Exception e) {
|
|
||||||
// build up a list of exceptions that occurred
|
|
||||||
exceptions.add(e);
|
|
||||||
|
|
||||||
String errorMessage = e.getLocalizedMessage();
|
|
||||||
if (errorMessage == null) {
|
|
||||||
// this is an unexpected condition, so dump the whole exception since
|
|
||||||
// it's probably a nasty internal error where the backtrace would be
|
|
||||||
// useful
|
|
||||||
errorMessage = StringUtils.stringifyException(e);
|
|
||||||
LOG.debug(errorMessage);
|
|
||||||
} else {
|
|
||||||
errorMessage = errorMessage.split("\n", 2)[0];
|
|
||||||
}
|
|
||||||
displayError(errorMessage);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Display an error string prefaced with the command name. Also increments
|
|
||||||
* the error count for the command which will result in a non-zero exit
|
|
||||||
* code.
|
|
||||||
* @param message error message to display
|
|
||||||
*/
|
|
||||||
public void displayError(String message) {
|
|
||||||
numErrors++;
|
|
||||||
displayWarning(message);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Display an warning string prefaced with the command name.
|
|
||||||
* @param message warning message to display
|
|
||||||
*/
|
|
||||||
public void displayWarning(String message) {
|
|
||||||
String cmd = cmdName.startsWith("-") ? cmdName.substring(1) : cmdName;
|
|
||||||
err.println(cmd + ": " + message);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -17,22 +17,19 @@
|
||||||
*/
|
*/
|
||||||
package org.apache.hadoop.fs.shell;
|
package org.apache.hadoop.fs.shell;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.*;
|
||||||
import java.util.LinkedList;
|
import java.util.List;
|
||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.classification.InterfaceStability;
|
import org.apache.hadoop.fs.FileSystem;
|
||||||
import org.apache.hadoop.fs.ContentSummary;
|
import org.apache.hadoop.fs.Path;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Count the number of directories, files, bytes, quota, and remaining quota.
|
* Count the number of directories, files, bytes, quota, and remaining quota.
|
||||||
*/
|
*/
|
||||||
@InterfaceAudience.Private
|
public class Count extends Command {
|
||||||
@InterfaceStability.Evolving
|
|
||||||
|
|
||||||
public class Count extends FsCommand {
|
|
||||||
public static final String NAME = "count";
|
public static final String NAME = "count";
|
||||||
public static final String USAGE = "-" + NAME + " [-q] <path> ...";
|
public static final String USAGE = "-" + NAME + "[-q] <path>";
|
||||||
public static final String DESCRIPTION = CommandUtils.formatDescription(USAGE,
|
public static final String DESCRIPTION = CommandUtils.formatDescription(USAGE,
|
||||||
"Count the number of directories, files and bytes under the paths",
|
"Count the number of directories, files and bytes under the paths",
|
||||||
"that match the specified file pattern. The output columns are:",
|
"that match the specified file pattern. The output columns are:",
|
||||||
|
@ -40,21 +37,41 @@ public class Count extends FsCommand {
|
||||||
"QUOTA REMAINING_QUATA SPACE_QUOTA REMAINING_SPACE_QUOTA ",
|
"QUOTA REMAINING_QUATA SPACE_QUOTA REMAINING_SPACE_QUOTA ",
|
||||||
" DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME");
|
" DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME");
|
||||||
|
|
||||||
private boolean showQuotas;
|
private boolean qOption;
|
||||||
|
|
||||||
@Override
|
/** Constructor
|
||||||
protected void processOptions(LinkedList<String> args) {
|
*
|
||||||
CommandFormat cf = new CommandFormat(NAME, 1, Integer.MAX_VALUE, "q");
|
* @param cmd the count command
|
||||||
cf.parse(args);
|
* @param pos the starting index of the arguments
|
||||||
if (args.size() == 0) { // default path is the current working directory
|
*/
|
||||||
args.add(".");
|
public Count(String[] cmd, int pos, Configuration conf) {
|
||||||
|
super(conf);
|
||||||
|
CommandFormat c = new CommandFormat(NAME, 1, Integer.MAX_VALUE, "q");
|
||||||
|
List<String> parameters = c.parse(cmd, pos);
|
||||||
|
this.args = parameters.toArray(new String[parameters.size()]);
|
||||||
|
if (this.args.length == 0) { // default path is the current working directory
|
||||||
|
this.args = new String[] {"."};
|
||||||
}
|
}
|
||||||
showQuotas = cf.getOpt("q");
|
this.qOption = c.getOpt("q") ? true: false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Check if a command is the count command
|
||||||
|
*
|
||||||
|
* @param cmd A string representation of a command starting with "-"
|
||||||
|
* @return true if this is a count command; false otherwise
|
||||||
|
*/
|
||||||
|
public static boolean matches(String cmd) {
|
||||||
|
return ("-" + NAME).equals(cmd);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void processPath(PathData src) throws IOException {
|
public String getCommandName() {
|
||||||
ContentSummary summary = src.fs.getContentSummary(src.path);
|
return NAME;
|
||||||
out.println(summary.toString(showQuotas) + src.path);
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void run(Path path) throws IOException {
|
||||||
|
FileSystem fs = path.getFileSystem(getConf());
|
||||||
|
System.out.println(fs.getContentSummary(path).toString(qOption) + path);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,46 +0,0 @@
|
||||||
/**
|
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one
|
|
||||||
* or more contributor license agreements. See the NOTICE file
|
|
||||||
* distributed with this work for additional information
|
|
||||||
* regarding copyright ownership. The ASF licenses this file
|
|
||||||
* to you under the Apache License, Version 2.0 (the
|
|
||||||
* "License"); you may not use this file except in compliance
|
|
||||||
* with the License. You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package org.apache.hadoop.fs.shell;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
|
||||||
import org.apache.hadoop.classification.InterfaceStability;
|
|
||||||
import org.apache.hadoop.fs.Path;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Base class for all "hadoop fs" commands
|
|
||||||
*/
|
|
||||||
|
|
||||||
@InterfaceAudience.Private
|
|
||||||
@InterfaceStability.Evolving
|
|
||||||
|
|
||||||
// this class may not look useful now, but it's a placeholder for future
|
|
||||||
// functionality to act as a registry for fs commands. currently it's being
|
|
||||||
// used to implement unnecessary abstract methods in the base class
|
|
||||||
|
|
||||||
abstract public class FsCommand extends Command {
|
|
||||||
public String getCommandName() {
|
|
||||||
return cmdName;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected void run(Path path) throws IOException {
|
|
||||||
throw new RuntimeException("not supposed to get here");
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,123 +0,0 @@
|
||||||
/**
|
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one
|
|
||||||
* or more contributor license agreements. See the NOTICE file
|
|
||||||
* distributed with this work for additional information
|
|
||||||
* regarding copyright ownership. The ASF licenses this file
|
|
||||||
* to you under the Apache License, Version 2.0 (the
|
|
||||||
* "License"); you may not use this file except in compliance
|
|
||||||
* with the License. You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package org.apache.hadoop.fs.shell;
|
|
||||||
|
|
||||||
import java.io.FileNotFoundException;
|
|
||||||
import java.io.IOException;
|
|
||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
|
||||||
import org.apache.hadoop.classification.InterfaceStability;
|
|
||||||
import org.apache.hadoop.fs.FileStatus;
|
|
||||||
import org.apache.hadoop.fs.FileSystem;
|
|
||||||
import org.apache.hadoop.fs.Path;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Encapsulates a Path (path), its FileStatus (stat), and its FileSystem (fs).
|
|
||||||
* The stat field will be null if the path does not exist.
|
|
||||||
*/
|
|
||||||
@InterfaceAudience.Private
|
|
||||||
@InterfaceStability.Evolving
|
|
||||||
|
|
||||||
public class PathData {
|
|
||||||
public final Path path;
|
|
||||||
public FileStatus stat;
|
|
||||||
public final FileSystem fs;
|
|
||||||
public boolean exists;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Creates an object to wrap the given parameters as fields.
|
|
||||||
* @param theFs the FileSystem
|
|
||||||
* @param thePath a Path
|
|
||||||
* @param theStat the FileStatus (may be null if the path doesn't exist)
|
|
||||||
*/
|
|
||||||
public PathData(FileSystem theFs, Path thePath, FileStatus theStat) {
|
|
||||||
path = thePath;
|
|
||||||
stat = theStat;
|
|
||||||
fs = theFs;
|
|
||||||
exists = (stat != null);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Convenience ctor that looks up the file status for a path. If the path
|
|
||||||
* doesn't exist, then the status will be null
|
|
||||||
* @param fs the FileSystem for the path
|
|
||||||
* @param path the pathname to lookup
|
|
||||||
* @throws IOException
|
|
||||||
*/
|
|
||||||
public PathData(FileSystem fs, Path path) throws IOException {
|
|
||||||
this(fs, path, getStat(fs, path));
|
|
||||||
}
|
|
||||||
|
|
||||||
// need a static method for the ctor above
|
|
||||||
private static FileStatus getStat(FileSystem fs, Path path)
|
|
||||||
throws IOException {
|
|
||||||
FileStatus status = null;
|
|
||||||
try {
|
|
||||||
status = fs.getFileStatus(path);
|
|
||||||
} catch (FileNotFoundException e) {} // ignore FNF
|
|
||||||
return status;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Convenience ctor that extracts the path from the given file status
|
|
||||||
* @param fs the FileSystem for the FileStatus
|
|
||||||
* @param stat the FileStatus
|
|
||||||
*/
|
|
||||||
public PathData(FileSystem fs, FileStatus stat) {
|
|
||||||
this(fs, stat.getPath(), stat);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Updates the paths's file status
|
|
||||||
* @return the updated FileStatus
|
|
||||||
* @throws IOException if anything goes wrong...
|
|
||||||
*/
|
|
||||||
public FileStatus refreshStatus() throws IOException {
|
|
||||||
stat = fs.getFileStatus(path);
|
|
||||||
return stat;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Returns a list of PathData objects of the items contained in the given
|
|
||||||
* directory.
|
|
||||||
* @return list of PathData objects for its children
|
|
||||||
* @throws IOException if anything else goes wrong...
|
|
||||||
*/
|
|
||||||
public PathData[] getDirectoryContents() throws IOException {
|
|
||||||
if (!stat.isDirectory()) {
|
|
||||||
throw new IOException(path + ": Not a directory");
|
|
||||||
}
|
|
||||||
|
|
||||||
FileStatus[] stats = fs.listStatus(path);
|
|
||||||
PathData[] items = new PathData[stats.length];
|
|
||||||
for (int i=0; i < stats.length; i++) {
|
|
||||||
items[i] = new PathData(fs, stats[i]);
|
|
||||||
}
|
|
||||||
return items;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Returns the printable version of the path that is just the
|
|
||||||
* filesystem path instead of the full uri
|
|
||||||
* @return String of the path
|
|
||||||
*/
|
|
||||||
public String toString() {
|
|
||||||
return path.toUri().getPath();
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,47 +0,0 @@
|
||||||
/**
|
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one
|
|
||||||
* or more contributor license agreements. See the NOTICE file
|
|
||||||
* distributed with this work for additional information
|
|
||||||
* regarding copyright ownership. The ASF licenses this file
|
|
||||||
* to you under the Apache License, Version 2.0 (the
|
|
||||||
* "License"); you may not use this file except in compliance
|
|
||||||
* with the License. You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
package org.apache.hadoop.fs.shell;
|
|
||||||
|
|
||||||
import org.apache.hadoop.conf.Configuration;
|
|
||||||
import org.apache.hadoop.fs.FileSystem;
|
|
||||||
import org.apache.hadoop.fs.Path;
|
|
||||||
import org.junit.BeforeClass;
|
|
||||||
import org.junit.Test;
|
|
||||||
import static org.junit.Assert.*;
|
|
||||||
|
|
||||||
public class TestPathData {
|
|
||||||
protected static Configuration conf;
|
|
||||||
protected static FileSystem fs;
|
|
||||||
protected static Path dir;
|
|
||||||
|
|
||||||
@BeforeClass
|
|
||||||
public static void initialize() throws Exception {
|
|
||||||
conf = new Configuration();
|
|
||||||
fs = FileSystem.getLocal(conf);
|
|
||||||
dir = new Path("/tmp");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testWithPath() throws Exception {
|
|
||||||
PathData item = new PathData(fs, dir);
|
|
||||||
assertEquals(fs, item.fs);
|
|
||||||
assertEquals(dir, item.path);
|
|
||||||
assertTrue(item.stat != null);
|
|
||||||
assertTrue(item.stat.isDirectory());
|
|
||||||
}
|
|
||||||
}
|
|
Loading…
Reference in New Issue