From d358eb75b79b17f85ae9fd831a0bd065b87bf924 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Wed, 13 Apr 2011 20:23:51 +0000 Subject: [PATCH] HADOOP-7224. Add CommandFactory to shell. Contributed by Daryn Sharp git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1091902 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES.txt | 2 + src/java/org/apache/hadoop/fs/FsShell.java | 74 +++++++--- .../org/apache/hadoop/fs/shell/Command.java | 31 +++++ .../hadoop/fs/shell/CommandFactory.java | 130 ++++++++++++++++++ .../org/apache/hadoop/fs/shell/Count.java | 8 ++ .../org/apache/hadoop/fs/shell/FsCommand.java | 8 ++ 6 files changed, 231 insertions(+), 22 deletions(-) create mode 100644 src/java/org/apache/hadoop/fs/shell/CommandFactory.java diff --git a/CHANGES.txt b/CHANGES.txt index f9111547c71..d7ce78c532b 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -89,6 +89,8 @@ Trunk (unreleased changes) HADOOP-7223. FileContext createFlag combinations are not clearly defined. (suresh) + HADOOP-7224. Add CommandFactory to shell. (Daryn Sharp via szetszwo) + OPTIMIZATIONS BUG FIXES diff --git a/src/java/org/apache/hadoop/fs/FsShell.java b/src/java/org/apache/hadoop/fs/FsShell.java index 1734103ec0a..d45a68ed8dc 100644 --- a/src/java/org/apache/hadoop/fs/FsShell.java +++ b/src/java/org/apache/hadoop/fs/FsShell.java @@ -26,7 +26,6 @@ import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; -import java.util.LinkedList; import java.util.List; import java.util.TimeZone; import java.util.zip.GZIPInputStream; @@ -36,8 +35,10 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.shell.Command; +import org.apache.hadoop.fs.shell.CommandFactory; import org.apache.hadoop.fs.shell.CommandFormat; -import org.apache.hadoop.fs.shell.Count; +import org.apache.hadoop.fs.shell.FsCommand; import org.apache.hadoop.io.DataInputBuffer; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.IOUtils; @@ -61,6 +62,8 @@ public class FsShell extends Configured implements Tool { protected FileSystem fs; private Trash trash; + protected CommandFactory commandFactory; + public static final SimpleDateFormat dateForm = new SimpleDateFormat("yyyy-MM-dd HH:mm"); protected static final SimpleDateFormat modifFmt = @@ -86,6 +89,7 @@ public class FsShell extends Configured implements Tool { super(conf); fs = null; trash = null; + commandFactory = new CommandFactory(); } protected void init() throws IOException { @@ -1398,9 +1402,7 @@ public class FsShell extends Configured implements Tool { "[-tail [-f] ] [-text ]\n\t" + "[" + FsShellPermissions.CHMOD_USAGE + "]\n\t" + "[" + FsShellPermissions.CHOWN_USAGE + "]\n\t" + - "[" + FsShellPermissions.CHGRP_USAGE + "]\n\t" + - "[" + Count.USAGE + "]\n\t" + - "[-help [cmd]]\n"; + "[" + FsShellPermissions.CHGRP_USAGE + "]"; String conf ="-conf : Specify an application configuration file."; @@ -1559,7 +1561,10 @@ public class FsShell extends Configured implements Tool { String help = "-help [cmd]: \tDisplays help for given command or all commands if none\n" + "\t\tis specified.\n"; - if ("fs".equals(cmd)) { + Command instance = commandFactory.getInstance("-" + cmd); + if (instance != null) { + System.out.println(instance.getDescription()); + } else if ("fs".equals(cmd)) { System.out.println(fs); } else if ("conf".equals(cmd)) { System.out.println(conf); @@ -1623,12 +1628,16 @@ public class FsShell extends Configured implements Tool { System.out.println(chown); } else if ("chgrp".equals(cmd)) { System.out.println(chgrp); - } else if (Count.NAME.equals(cmd)) { - System.out.println(Count.DESCRIPTION); } else if ("help".equals(cmd)) { System.out.println(help); } else { System.out.println(summary); + for (String thisCmdName : commandFactory.getNames()) { + instance = commandFactory.getInstance(thisCmdName); + System.out.println(instance.getUsage()); + } + System.out.println("\t[-help [cmd]]\n"); + System.out.println(fs); System.out.println(ls); System.out.println(lsr); @@ -1657,7 +1666,12 @@ public class FsShell extends Configured implements Tool { System.out.println(chmod); System.out.println(chown); System.out.println(chgrp); - System.out.println(Count.DESCRIPTION); + + for (String thisCmdName : commandFactory.getNames()) { + instance = commandFactory.getInstance(thisCmdName); + System.out.println(instance.getDescription()); + } + System.out.println(help); } } @@ -1742,9 +1756,13 @@ public class FsShell extends Configured implements Tool { * Displays format of commands. * */ - private static void printUsage(String cmd) { + private void printUsage(String cmd) { String prefix = "Usage: java " + FsShell.class.getSimpleName(); - if ("-fs".equals(cmd)) { + + Command instance = commandFactory.getInstance(cmd); + if (instance != null) { + System.err.println(prefix + " [" + instance.getUsage() + "]"); + } else if ("-fs".equals(cmd)) { System.err.println("Usage: java FsShell" + " [-fs ]"); } else if ("-conf".equals(cmd)) { @@ -1762,8 +1780,6 @@ public class FsShell extends Configured implements Tool { } else if ("-df".equals(cmd) ) { System.err.println("Usage: java FsShell" + " [" + cmd + " []]"); - } else if ("-count".equals(cmd)) { - System.err.println(prefix + " [" + Count.USAGE + "]"); } else if ("-rm".equals(cmd) || "-rmr".equals(cmd)) { System.err.println("Usage: java FsShell [" + cmd + " [-skipTrash] ]"); @@ -1801,7 +1817,6 @@ public class FsShell extends Configured implements Tool { System.err.println(" [-df []]"); System.err.println(" [-du [-s] [-h] ]"); System.err.println(" [-dus ]"); - System.err.println(" [" + Count.USAGE + "]"); System.err.println(" [-mv ]"); System.err.println(" [-cp ]"); System.err.println(" [-rm [-skipTrash] ]"); @@ -1825,6 +1840,10 @@ public class FsShell extends Configured implements Tool { System.err.println(" [" + FsShellPermissions.CHMOD_USAGE + "]"); System.err.println(" [" + FsShellPermissions.CHOWN_USAGE + "]"); System.err.println(" [" + FsShellPermissions.CHGRP_USAGE + "]"); + for (String name : commandFactory.getNames()) { + instance = commandFactory.getInstance(name); + System.err.println(" [" + instance.getUsage() + "]"); + } System.err.println(" [-help [cmd]]"); System.err.println(); ToolRunner.printGenericCommandUsage(System.err); @@ -1835,7 +1854,12 @@ public class FsShell extends Configured implements Tool { * run */ public int run(String argv[]) throws Exception { - + // TODO: This isn't the best place, but this class is being abused with + // subclasses which of course override this method. There really needs + // to be a better base class for all commands + commandFactory.setConf(getConf()); + commandFactory.registerCommands(FsCommand.class); + if (argv.length < 1) { printUsage(""); return -1; @@ -1890,7 +1914,19 @@ public class FsShell extends Configured implements Tool { exitCode = 0; try { - if ("-put".equals(cmd) || "-copyFromLocal".equals(cmd)) { + Command instance = commandFactory.getInstance(cmd); + if (instance != null) { + try { + exitCode = instance.run(Arrays.copyOfRange(argv, i, argv.length)); + } catch (Exception e) { + exitCode = -1; + LOG.debug("Error", e); + instance.displayError(e); + if (e instanceof IllegalArgumentException) { + printUsage(cmd); + } + } + } else if ("-put".equals(cmd) || "-copyFromLocal".equals(cmd)) { Path[] srcs = new Path[argv.length-2]; for (int j=0 ; i < argv.length-1 ;) srcs[j++] = new Path(argv[i++]); @@ -1951,12 +1987,6 @@ public class FsShell extends Configured implements Tool { du(argv, i); } else if ("-dus".equals(cmd)) { dus(argv, i); - } else if ("-count".equals(cmd)) { - // TODO: next two lines are a temporary crutch until this entire - // block is overhauled - Count runner = ReflectionUtils.newInstance(Count.class, getConf()); - runner.setCommandName(cmd); // TODO: will change with factory - exitCode = runner.run(Arrays.copyOfRange(argv, 1, argv.length)); } else if ("-mkdir".equals(cmd)) { exitCode = doall(cmd, argv, i); } else if ("-touchz".equals(cmd)) { diff --git a/src/java/org/apache/hadoop/fs/shell/Command.java b/src/java/org/apache/hadoop/fs/shell/Command.java index 81461cfe7d8..0c0b18dac33 100644 --- a/src/java/org/apache/hadoop/fs/shell/Command.java +++ b/src/java/org/apache/hadoop/fs/shell/Command.java @@ -365,4 +365,35 @@ abstract public class Command extends Configured { public void displayWarning(String message) { err.println(getCommandName() + ": " + message); } + + /** + * The short usage suitable for the synopsis + * @return "name options" + */ + public String getUsage() { + return getCommandField("USAGE"); + } + + /** + * The long usage suitable for help output + * @return text of the usage + */ + public String getDescription() { + return getCommandField("DESCRIPTION"); + } + + /** + * Get a public static class field + * @param field the field to retrieve + * @return String of the field + */ + private String getCommandField(String field) { + String value; + try { + value = (String)this.getClass().getField(field).get(null); + } catch (Exception e) { + throw new RuntimeException(StringUtils.stringifyException(e)); + } + return value; + } } diff --git a/src/java/org/apache/hadoop/fs/shell/CommandFactory.java b/src/java/org/apache/hadoop/fs/shell/CommandFactory.java new file mode 100644 index 00000000000..c8425145cc0 --- /dev/null +++ b/src/java/org/apache/hadoop/fs/shell/CommandFactory.java @@ -0,0 +1,130 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.shell; + +import java.util.Arrays; +import java.util.Hashtable; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.util.StringUtils; + +/** class to search for and register commands */ + +@InterfaceAudience.Private +@InterfaceStability.Unstable + +public class CommandFactory extends Configured implements Configurable { + private Hashtable> classMap = + new Hashtable>(); + + /** Factory constructor for commands */ + public CommandFactory() { + this(null); + } + + /** + * Factory constructor for commands + * @param conf the hadoop configuration + */ + public CommandFactory(Configuration conf) { + super(conf); + } + + /** + * Invokes "static void registerCommands(CommandFactory)" on the given class. + * This method abstracts the contract between the factory and the command + * class. Do not assume that directly invoking registerCommands on the + * given class will have the same effect. + * @param registrarClass class to allow an opportunity to register + */ + public void registerCommands(Class registrarClass) { + try { + registrarClass.getMethod( + "registerCommands", CommandFactory.class + ).invoke(null, this); + } catch (Exception e) { + throw new RuntimeException(StringUtils.stringifyException(e)); + } + } + + /** + * Register the given class as handling the given list of command + * names. + * @param cmdClass the class implementing the command names + * @param names one or more command names that will invoke this class + */ + public void addClass(Class cmdClass, String ... names) { + for (String name : names) classMap.put(name, cmdClass); + } + + /** + * Returns the class implementing the given command. The + * class must have been registered via + * {@link #addClass(Class, String...)} + * @param cmd name of the command + * @return instance of the requested command + */ + protected Class getClass(String cmd) { + return classMap.get(cmd); + } + + /** + * Returns an instance of the class implementing the given command. The + * class must have been registered via + * {@link #addClass(Class, String...)} + * @param cmd name of the command + * @return instance of the requested command + */ + public Command getInstance(String cmd) { + return getInstance(cmd, getConf()); + } + + /** + * Get an instance of the requested command + * @param cmdName name of the command to lookup + * @param conf the hadoop configuration + * @return the {@link Command} or null if the command is unknown + */ + public Command getInstance(String cmdName, Configuration conf) { + if (conf == null) throw new NullPointerException("configuration is null"); + + Command instance = null; + Class cmdClass = getClass(cmdName); + if (cmdClass != null) { + instance = ReflectionUtils.newInstance(cmdClass, conf); + instance.setCommandName(cmdName); + } + return instance; + } + + /** + * Gets all of the registered commands + * @return a sorted list of command names + */ + public String[] getNames() { + String[] names = classMap.keySet().toArray(new String[0]); + Arrays.sort(names); + return names; + } +} diff --git a/src/java/org/apache/hadoop/fs/shell/Count.java b/src/java/org/apache/hadoop/fs/shell/Count.java index 6e2991d741e..93a04178e82 100644 --- a/src/java/org/apache/hadoop/fs/shell/Count.java +++ b/src/java/org/apache/hadoop/fs/shell/Count.java @@ -34,6 +34,14 @@ import org.apache.hadoop.fs.FsShell; @InterfaceStability.Evolving public class Count extends FsCommand { + /** + * Register the names for the count command + * @param factory the command factory that will instantiate this class + */ + public static void registerCommands(CommandFactory factory) { + factory.addClass(Count.class, "-count"); + } + public static final String NAME = "count"; public static final String USAGE = "-" + NAME + "[-q] "; public static final String DESCRIPTION = CommandUtils.formatDescription(USAGE, diff --git a/src/java/org/apache/hadoop/fs/shell/FsCommand.java b/src/java/org/apache/hadoop/fs/shell/FsCommand.java index 29ba4da050f..db7a7a75969 100644 --- a/src/java/org/apache/hadoop/fs/shell/FsCommand.java +++ b/src/java/org/apache/hadoop/fs/shell/FsCommand.java @@ -37,6 +37,14 @@ import org.apache.hadoop.fs.Path; // used to implement unnecessary abstract methods in the base class abstract public class FsCommand extends Command { + /** + * Register the command classes used by the fs subcommand + * @param factory where to register the class + */ + public static void registerCommands(CommandFactory factory) { + Count.registerCommands(factory); + } + protected FsCommand() {} protected FsCommand(Configuration conf) {