Revert HADOOP-7202 since HDFS cannot be compiled.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1086455 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Tsz-wo Sze 2011-03-29 00:39:58 +00:00
parent 50b1f9fc73
commit bb4f277407
7 changed files with 66 additions and 539 deletions

View File

@ -84,8 +84,6 @@ Trunk (unreleased changes)
HADOOP-7180. Better support on CommandFormat on the API and exceptions. HADOOP-7180. Better support on CommandFormat on the API and exceptions.
(Daryn Sharp via szetszwo) (Daryn Sharp via szetszwo)
HADOOP-7202. Improve shell Command base class. (Daryn Sharp via szetszwo)
OPTIMIZATIONS OPTIMIZATIONS
HADOOP-6949. Reduce RPC packet size of primitive arrays using HADOOP-6949. Reduce RPC packet size of primitive arrays using

View File

@ -370,7 +370,7 @@ public class FsShell extends Configured implements Tool {
private class TextRecordInputStream extends InputStream { private class TextRecordInputStream extends InputStream {
SequenceFile.Reader r; SequenceFile.Reader r;
WritableComparable<?> key; WritableComparable key;
Writable val; Writable val;
DataInputBuffer inbuf; DataInputBuffer inbuf;
@ -1691,6 +1691,8 @@ public class FsShell extends Configured implements Tool {
delete(argv[i], true, rmSkipTrash); delete(argv[i], true, rmSkipTrash);
} else if ("-df".equals(cmd)) { } else if ("-df".equals(cmd)) {
df(argv[i]); df(argv[i]);
} else if (Count.matches(cmd)) {
new Count(argv, i, getConf()).runAll();
} else if ("-ls".equals(cmd)) { } else if ("-ls".equals(cmd)) {
exitCode = ls(argv[i], false); exitCode = ls(argv[i], false);
} else if ("-lsr".equals(cmd)) { } else if ("-lsr".equals(cmd)) {
@ -1757,7 +1759,7 @@ public class FsShell extends Configured implements Tool {
} else if ("-df".equals(cmd) ) { } else if ("-df".equals(cmd) ) {
System.err.println("Usage: java FsShell" + System.err.println("Usage: java FsShell" +
" [" + cmd + " [<path>]]"); " [" + cmd + " [<path>]]");
} else if ("-count".equals(cmd)) { } else if (Count.matches(cmd)) {
System.err.println(prefix + " [" + Count.USAGE + "]"); System.err.println(prefix + " [" + Count.USAGE + "]");
} else if ("-rm".equals(cmd) || "-rmr".equals(cmd)) { } else if ("-rm".equals(cmd) || "-rmr".equals(cmd)) {
System.err.println("Usage: java FsShell [" + cmd + System.err.println("Usage: java FsShell [" + cmd +
@ -1946,13 +1948,8 @@ public class FsShell extends Configured implements Tool {
du(argv, i); du(argv, i);
} else if ("-dus".equals(cmd)) { } else if ("-dus".equals(cmd)) {
dus(argv, i); dus(argv, i);
} else if ("-count".equals(cmd)) { } else if (Count.matches(cmd)) {
// TODO: next two lines are a temporary crutch until this entire exitCode = new Count(argv, i, getConf()).runAll();
// block is overhauled
LinkedList<String> args = new LinkedList<String>(Arrays.asList(argv));
args.removeFirst();
Count runner = ReflectionUtils.newInstance(Count.class, getConf());
exitCode = runner.run(cmd, args);
} else if ("-mkdir".equals(cmd)) { } else if ("-mkdir".equals(cmd)) {
exitCode = doall(cmd, argv, i); exitCode = doall(cmd, argv, i);
} else if ("-touchz".equals(cmd)) { } else if ("-touchz".equals(cmd)) {

View File

@ -17,53 +17,27 @@
*/ */
package org.apache.hadoop.fs.shell; package org.apache.hadoop.fs.shell;
import java.io.FileNotFoundException; import java.io.*;
import java.io.IOException;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured; import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.ipc.RemoteException;
/** /**
* An abstract class for the execution of a file system command * An abstract class for the execution of a file system command
*/ */
@InterfaceAudience.Private
@InterfaceStability.Evolving
abstract public class Command extends Configured { abstract public class Command extends Configured {
protected String[] args; protected String[] args;
protected String cmdName;
protected int exitCode = 0;
protected int numErrors = 0;
protected boolean recursive = false;
protected ArrayList<Exception> exceptions = new ArrayList<Exception>();
private static final Log LOG = LogFactory.getLog(Command.class);
/** allows stdout to be captured if necessary */
public PrintStream out = System.out;
/** allows stderr to be captured if necessary */
public PrintStream err = System.err;
protected Command() {}
/** Constructor */ /** Constructor */
protected Command(Configuration conf) { protected Command(Configuration conf) {
super(conf); super(conf);
} }
/** @return the command's name excluding the leading character - */ /** Return the command's name excluding the leading character - */
abstract public String getCommandName(); abstract public String getCommandName();
/** /**
@ -83,273 +57,30 @@ abstract public class Command extends Configured {
int exitCode = 0; int exitCode = 0;
for (String src : args) { for (String src : args) {
try { try {
List<PathData> srcs = expandGlob(src); Path srcPath = new Path(src);
for(PathData s : srcs) { FileSystem fs = srcPath.getFileSystem(getConf());
run(s.path); FileStatus[] statuses = fs.globStatus(srcPath);
if (statuses == null) {
System.err.println("Can not find listing for " + src);
exitCode = -1;
} else {
for(FileStatus s : statuses) {
run(s.getPath());
}
} }
} catch (RemoteException re) {
exitCode = -1;
String content = re.getLocalizedMessage();
int eol = content.indexOf('\n');
if (eol>=0) {
content = content.substring(0, eol);
}
System.err.println(getCommandName() + ": " + content);
} catch (IOException e) { } catch (IOException e) {
exitCode = -1; exitCode = -1;
displayError(e); System.err.println(getCommandName() + ": " + e.getLocalizedMessage());
} }
} }
return exitCode; return exitCode;
} }
/**
* Invokes the command handler. The default behavior is to process options,
* expand arguments, and then process each argument.
* <pre>
* run
* \-> {@link #processOptions(LinkedList)}
* \-> {@link #expandArguments(LinkedList)} -> {@link #expandArgument(String)}*
* \-> {@link #processArguments(LinkedList)}
* \-> {@link #processArgument(PathData)}*
* \-> {@link #processPathArgument(PathData)}
* \-> {@link #processPaths(PathData, PathData...)}
* \-> {@link #processPath(PathData)}*
* \-> {@link #processNonexistentPathArgument(PathData)}
* </pre>
* Most commands will chose to implement just
* {@link #processOptions(LinkedList)} and {@link #processPath(PathData)}
*
* @param cmd the command's name
* @param args the list of command line arguments
* @return the exit code for the command
* @throws IllegalArgumentException if called with invalid arguments
*/
public int run(String cmd, LinkedList<String> args) {
cmdName = cmd;
try {
processOptions(args);
processArguments(expandArguments(args));
} catch (IOException e) {
displayError(e);
}
return (numErrors == 0) ? exitCode : 1;
}
/**
* Must be implemented by commands to process the command line flags and
* check the bounds of the remaining arguments. If an
* IllegalArgumentException is thrown, the FsShell object will print the
* short usage of the command.
* @param args the command line arguments
* @throws IOException
*/
protected void processOptions(LinkedList<String> args) throws IOException {}
/**
* Expands a list of arguments into {@link PathData} objects. The default
* behavior is to call {@link #expandArgument(String)} on each element
* which by default globs the argument. The loop catches IOExceptions,
* increments the error count, and displays the exception.
* @param args strings to expand into {@link PathData} objects
* @return list of all {@link PathData} objects the arguments
* @throws IOException if anything goes wrong...
*/
protected LinkedList<PathData> expandArguments(LinkedList<String> args)
throws IOException {
LinkedList<PathData> expandedArgs = new LinkedList<PathData>();
for (String arg : args) {
try {
expandedArgs.addAll(expandArgument(arg));
} catch (IOException e) { // other exceptions are probably nasty
displayError(e);
}
}
return expandedArgs;
}
/**
* Expand the given argument into a list of {@link PathData} objects.
* The default behavior is to expand globs. Commands may override to
* perform other expansions on an argument.
* @param arg string pattern to expand
* @return list of {@link PathData} objects
* @throws IOException if anything goes wrong...
*/
protected List<PathData> expandArgument(String arg) throws IOException {
return expandGlob(arg);
}
/**
* Processes the command's list of expanded arguments.
* {@link #processArgument(PathData)} will be invoked with each item
* in the list. The loop catches IOExceptions, increments the error
* count, and displays the exception.
* @param args a list of {@link PathData} to process
* @throws IOException if anything goes wrong...
*/
protected void processArguments(LinkedList<PathData> args)
throws IOException {
for (PathData arg : args) {
try {
processArgument(arg);
} catch (IOException e) {
displayError(e);
}
}
}
/**
* Processes a {@link PathData} item, calling
* {@link #processPathArgument(PathData)} or
* {@link #processNonexistentPathArgument(PathData)} on each item.
* @param arg {@link PathData} item to process
* @throws IOException if anything goes wrong...
*/
protected void processArgument(PathData item) throws IOException {
if (item.exists) {
processPathArgument(item);
} else {
processNonexistentPathArgument(item);
}
}
/**
* This is the last chance to modify an argument before going into the
* (possibly) recursive {@link #processPaths(PathData, PathData...)}
* -> {@link #processPath(PathData)} loop. Ex. ls and du use this to
* expand out directories.
* @param item a {@link PathData} representing a path which exists
* @throws IOException if anything goes wrong...
*/
protected void processPathArgument(PathData item) throws IOException {
// null indicates that the call is not via recursion, ie. there is
// no parent directory that was expanded
processPaths(null, item);
}
/**
* Provides a hook for handling paths that don't exist. By default it
* will throw an exception. Primarily overriden by commands that create
* paths such as mkdir or touch.
* @param item the {@link PathData} that doesn't exist
* @throws FileNotFoundException if arg is a path and it doesn't exist
* @throws IOException if anything else goes wrong...
*/
protected void processNonexistentPathArgument(PathData item)
throws IOException {
throw new FileNotFoundException(item + ": No such file or directory");
}
/**
* Iterates over the given expanded paths and invokes
* {@link #processPath(PathData)} on each element. If "recursive" is true,
* will do a post-visit DFS on directories.
* @param parent if called via a recurse, will be the parent dir, else null
* @param items a list of {@link PathData} objects to process
* @throws IOException if anything goes wrong...
*/
protected void processPaths(PathData parent, PathData ... items)
throws IOException {
// TODO: this really should be iterative
for (PathData item : items) {
try {
processPath(item);
if (recursive && item.stat.isDirectory()) {
recursePath(item);
}
} catch (IOException e) {
displayError(e);
}
}
}
/**
* Hook for commands to implement an operation to be applied on each
* path for the command. Note implementation of this method is optional
* if earlier methods in the chain handle the operation.
* @param item a {@link PathData} object
* @throws RuntimeException if invoked but not implemented
* @throws IOException if anything else goes wrong in the user-implementation
*/
protected void processPath(PathData item) throws IOException {
throw new RuntimeException("processPath() is not implemented");
}
/**
* Gets the directory listing for a path and invokes
* {@link #processPaths(PathData, PathData...)}
* @param item {@link PathData} for directory to recurse into
* @throws IOException if anything goes wrong...
*/
protected void recursePath(PathData item) throws IOException {
processPaths(item, item.getDirectoryContents());
}
/**
* Expand the given path as a glob pattern. Non-existent paths do not
* throw an exception because creation commands like touch and mkdir need
* to create them. The "stat" field will be null if the path does not
* exist.
* @param pattern the glob to expand
* @return list of {@link PathData} objects
* @throws FileNotFoundException the path is a glob with no matches
* @throws IOException anything else goes wrong...
*/
protected List<PathData> expandGlob(String pattern) throws IOException {
Path path = new Path(pattern);
FileSystem fs = path.getFileSystem(getConf());
FileStatus[] stats = fs.globStatus(path);
if (stats != null && stats.length == 0) { // glob failed to match
throw new FileNotFoundException(path + ": No such file or directory");
}
List<PathData> items = new LinkedList<PathData>();
if (stats == null) { // not a glob & file not found, so null stat block
items.add(new PathData(fs, path, null));
} else {
// convert all the stats to PathData objs
for (FileStatus stat : stats) {
items.add(new PathData(fs, stat));
}
}
return items;
}
/**
* Display an exception prefaced with the command name. Also increments
* the error count for the command which will result in a non-zero exit
* code.
* @param e exception to display
*/
public void displayError(Exception e) {
// build up a list of exceptions that occurred
exceptions.add(e);
String errorMessage = e.getLocalizedMessage();
if (errorMessage == null) {
// this is an unexpected condition, so dump the whole exception since
// it's probably a nasty internal error where the backtrace would be
// useful
errorMessage = StringUtils.stringifyException(e);
LOG.debug(errorMessage);
} else {
errorMessage = errorMessage.split("\n", 2)[0];
}
displayError(errorMessage);
}
/**
* Display an error string prefaced with the command name. Also increments
* the error count for the command which will result in a non-zero exit
* code.
* @param message error message to display
*/
public void displayError(String message) {
numErrors++;
displayWarning(message);
}
/**
* Display an warning string prefaced with the command name.
* @param message warning message to display
*/
public void displayWarning(String message) {
String cmd = cmdName.startsWith("-") ? cmdName.substring(1) : cmdName;
err.println(cmd + ": " + message);
}
} }

View File

@ -17,22 +17,19 @@
*/ */
package org.apache.hadoop.fs.shell; package org.apache.hadoop.fs.shell;
import java.io.IOException; import java.io.*;
import java.util.LinkedList; import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.Path;
/** /**
* Count the number of directories, files, bytes, quota, and remaining quota. * Count the number of directories, files, bytes, quota, and remaining quota.
*/ */
@InterfaceAudience.Private public class Count extends Command {
@InterfaceStability.Evolving
public class Count extends FsCommand {
public static final String NAME = "count"; public static final String NAME = "count";
public static final String USAGE = "-" + NAME + " [-q] <path> ..."; public static final String USAGE = "-" + NAME + "[-q] <path>";
public static final String DESCRIPTION = CommandUtils.formatDescription(USAGE, public static final String DESCRIPTION = CommandUtils.formatDescription(USAGE,
"Count the number of directories, files and bytes under the paths", "Count the number of directories, files and bytes under the paths",
"that match the specified file pattern. The output columns are:", "that match the specified file pattern. The output columns are:",
@ -40,21 +37,41 @@ public class Count extends FsCommand {
"QUOTA REMAINING_QUATA SPACE_QUOTA REMAINING_SPACE_QUOTA ", "QUOTA REMAINING_QUATA SPACE_QUOTA REMAINING_SPACE_QUOTA ",
" DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME"); " DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME");
private boolean showQuotas; private boolean qOption;
@Override /** Constructor
protected void processOptions(LinkedList<String> args) { *
CommandFormat cf = new CommandFormat(NAME, 1, Integer.MAX_VALUE, "q"); * @param cmd the count command
cf.parse(args); * @param pos the starting index of the arguments
if (args.size() == 0) { // default path is the current working directory */
args.add("."); public Count(String[] cmd, int pos, Configuration conf) {
super(conf);
CommandFormat c = new CommandFormat(NAME, 1, Integer.MAX_VALUE, "q");
List<String> parameters = c.parse(cmd, pos);
this.args = parameters.toArray(new String[parameters.size()]);
if (this.args.length == 0) { // default path is the current working directory
this.args = new String[] {"."};
} }
showQuotas = cf.getOpt("q"); this.qOption = c.getOpt("q") ? true: false;
}
/** Check if a command is the count command
*
* @param cmd A string representation of a command starting with "-"
* @return true if this is a count command; false otherwise
*/
public static boolean matches(String cmd) {
return ("-" + NAME).equals(cmd);
} }
@Override @Override
protected void processPath(PathData src) throws IOException { public String getCommandName() {
ContentSummary summary = src.fs.getContentSummary(src.path); return NAME;
out.println(summary.toString(showQuotas) + src.path); }
@Override
protected void run(Path path) throws IOException {
FileSystem fs = path.getFileSystem(getConf());
System.out.println(fs.getContentSummary(path).toString(qOption) + path);
} }
} }

View File

@ -1,46 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.shell;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.Path;
/**
* Base class for all "hadoop fs" commands
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
// this class may not look useful now, but it's a placeholder for future
// functionality to act as a registry for fs commands. currently it's being
// used to implement unnecessary abstract methods in the base class
abstract public class FsCommand extends Command {
public String getCommandName() {
return cmdName;
}
protected void run(Path path) throws IOException {
throw new RuntimeException("not supposed to get here");
}
}

View File

@ -1,123 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.shell;
import java.io.FileNotFoundException;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
/**
* Encapsulates a Path (path), its FileStatus (stat), and its FileSystem (fs).
* The stat field will be null if the path does not exist.
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
public class PathData {
public final Path path;
public FileStatus stat;
public final FileSystem fs;
public boolean exists;
/**
* Creates an object to wrap the given parameters as fields.
* @param theFs the FileSystem
* @param thePath a Path
* @param theStat the FileStatus (may be null if the path doesn't exist)
*/
public PathData(FileSystem theFs, Path thePath, FileStatus theStat) {
path = thePath;
stat = theStat;
fs = theFs;
exists = (stat != null);
}
/**
* Convenience ctor that looks up the file status for a path. If the path
* doesn't exist, then the status will be null
* @param fs the FileSystem for the path
* @param path the pathname to lookup
* @throws IOException
*/
public PathData(FileSystem fs, Path path) throws IOException {
this(fs, path, getStat(fs, path));
}
// need a static method for the ctor above
private static FileStatus getStat(FileSystem fs, Path path)
throws IOException {
FileStatus status = null;
try {
status = fs.getFileStatus(path);
} catch (FileNotFoundException e) {} // ignore FNF
return status;
}
/**
* Convenience ctor that extracts the path from the given file status
* @param fs the FileSystem for the FileStatus
* @param stat the FileStatus
*/
public PathData(FileSystem fs, FileStatus stat) {
this(fs, stat.getPath(), stat);
}
/**
* Updates the paths's file status
* @return the updated FileStatus
* @throws IOException if anything goes wrong...
*/
public FileStatus refreshStatus() throws IOException {
stat = fs.getFileStatus(path);
return stat;
}
/**
* Returns a list of PathData objects of the items contained in the given
* directory.
* @return list of PathData objects for its children
* @throws IOException if anything else goes wrong...
*/
public PathData[] getDirectoryContents() throws IOException {
if (!stat.isDirectory()) {
throw new IOException(path + ": Not a directory");
}
FileStatus[] stats = fs.listStatus(path);
PathData[] items = new PathData[stats.length];
for (int i=0; i < stats.length; i++) {
items[i] = new PathData(fs, stats[i]);
}
return items;
}
/**
* Returns the printable version of the path that is just the
* filesystem path instead of the full uri
* @return String of the path
*/
public String toString() {
return path.toUri().getPath();
}
}

View File

@ -1,47 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.shell;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.BeforeClass;
import org.junit.Test;
import static org.junit.Assert.*;
public class TestPathData {
protected static Configuration conf;
protected static FileSystem fs;
protected static Path dir;
@BeforeClass
public static void initialize() throws Exception {
conf = new Configuration();
fs = FileSystem.getLocal(conf);
dir = new Path("/tmp");
}
@Test
public void testWithPath() throws Exception {
PathData item = new PathData(fs, dir);
assertEquals(fs, item.fs);
assertEquals(dir, item.path);
assertTrue(item.stat != null);
assertTrue(item.stat.isDirectory());
}
}