YARN-2616 [YARN-913] Add CLI client to the registry to list, view and manipulate entries. (Akshay Radia via stevel)

hadoop-yarn-project/CHANGES.txt

@@ -44,6 +44,9 @@ Release 2.7.0 - UNRELEASED
     YARN-2217. [YARN-1492] Shared cache client side changes. 
     (Chris Trezzo via kasha)
 
+     YARN-2616 [YARN-913] Add CLI client to the registry to list, view
+     and manipulate entries. (Akshay Radia via stevel)
+
   IMPROVEMENTS
 
     YARN-3005. [JDK7] Use switch statement for String instead of if-else

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/cli/RegistryCli.java

@@ -19,6 +19,7 @@
 
 import static org.apache.hadoop.registry.client.binding.RegistryTypeUtils.*;
 
+import java.io.Closeable;
 import java.io.IOException;
 import java.io.PrintStream;
 import java.net.URI;
@@ -34,12 +35,14 @@
 import org.apache.commons.cli.OptionBuilder;
 import org.apache.commons.cli.Options;
 import org.apache.commons.cli.ParseException;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.PathNotFoundException;
 import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.service.ServiceOperations;
+import org.apache.hadoop.util.ExitUtil;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.registry.client.api.BindFlags;
 import org.apache.hadoop.registry.client.api.RegistryOperations;
 import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
@@ -54,7 +57,10 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-public class RegistryCli extends Configured implements Tool {
+/**
+ * Command line for registry operations.
+ */
+public class RegistryCli extends Configured implements Tool, Closeable {
   private static final Logger LOG =
       LoggerFactory.getLogger(RegistryCli.class);
   protected final PrintStream sysout;
@@ -63,32 +69,64 @@ public class RegistryCli extends Configured implements Tool {
 
   private RegistryOperations registry;
 
-  static final String LS_USAGE = "ls pathName";
+  private static final String LS_USAGE = "ls pathName";
-  static final String RESOLVE_USAGE = "resolve pathName";
+  private static final String RESOLVE_USAGE = "resolve pathName";
-  static final String BIND_USAGE =
+  private static final String BIND_USAGE =
       "bind -inet  -api apiName -p portNumber -h hostName  pathName" + "\n"
       + "bind -webui uriString -api apiName  pathName" + "\n"
       + "bind -rest uriString -api apiName  pathName";
-  static final String MKNODE_USAGE = "mknode directoryName";
+  private static final String MKNODE_USAGE = "mknode directoryName";
-  static final String RM_USAGE = "rm pathName";
+  private static final String RM_USAGE = "rm pathName";
-  static final String USAGE =
+  private static final String USAGE =
       "\n" + LS_USAGE + "\n" + RESOLVE_USAGE + "\n" + BIND_USAGE + "\n" +
       MKNODE_USAGE + "\n" + RM_USAGE;
 
 
-
   public RegistryCli(PrintStream sysout, PrintStream syserr) {
-	    super(new YarnConfiguration());
+    Configuration conf = new Configuration();
+    super.setConf(conf);
+    registry = RegistryOperationsFactory.createInstance(conf);
+    registry.start();
     this.sysout = sysout;
     this.syserr = syserr;
   }
 
+  public RegistryCli(RegistryOperations reg,
+      Configuration conf,
+      PrintStream sysout,
+      PrintStream syserr) {
+    super(conf);
+    Preconditions.checkArgument(reg != null, "Null registry");
+    registry = reg;
+    this.sysout = sysout;
+    this.syserr = syserr;
+  }
 
   @SuppressWarnings("UseOfSystemOutOrSystemErr")
   public static void main(String[] args) throws Exception {
-    RegistryCli cli = new RegistryCli(System.out, System.err);
+    int res = -1;
-    int res = ToolRunner.run(cli, args);
+    try (RegistryCli cli = new RegistryCli(System.out, System.err)) {
-    System.exit(res);
+      res = ToolRunner.run(cli, args);
+    } catch (Exception e) {
+      ExitUtil.terminate(res, e);
+    }
+    ExitUtil.terminate(res);
+  }
+
+  /**
+   * Close the object by stopping the registry.
+   * <p>
+   * <i>Important:</i>
+   * <p>
+   *   After this call is made, no operations may be made of this
+   *   object, <i>or of a YARN registry instance used when constructing
+   *   this object. </i>
+   * @throws IOException
+   */
+  @Override
+  public void close() throws IOException {
+    ServiceOperations.stopQuietly(registry);
+    registry = null;
   }
 
   private int usageError(String err, String usage) {
@@ -104,30 +142,31 @@ private boolean validatePath(String path) {
     }
     return true;
   }
+
   @Override
   public int run(String[] args) throws Exception {
     Preconditions.checkArgument(getConf() != null, "null configuration");
-    registry = RegistryOperationsFactory.createInstance(
-        new YarnConfiguration(getConf()));
-    registry.start();
     if (args.length > 0) {
-      if (args[0].equals("ls")) {
+      switch (args[0]) {
+        case "ls":
           return ls(args);
-      } else if (args[0].equals("resolve")) {
+        case "resolve":
           return resolve(args);
-      } else if (args[0].equals("bind")) {
+        case "bind":
           return bind(args);
-      } else if (args[0].equals("mknode")) {
+        case "mknode":
           return mknode(args);
-      } else if (args[0].equals("rm")) {
+        case "rm":
           return rm(args);
-      }
+        default:
-    }
           return usageError("Invalid command: " + args[0], USAGE);
       }
+    }
+    return usageError("No command arg passed.", USAGE);
+  }
 
   @SuppressWarnings("unchecked")
-	public int ls(String [] args) {
+  public int ls(String[] args) {
 
     Options lsOption = new Options();
     CommandLineParser parser = new GnuParser();
@@ -138,8 +177,9 @@ public int ls(String [] args) {
       if (argsList.size() != 2) {
         return usageError("ls requires exactly one path argument", LS_USAGE);
       }
-			if (!validatePath(argsList.get(1)))
+      if (!validatePath(argsList.get(1))) {
         return -1;
+      }
 
       try {
         List<String> children = registry.list(argsList.get(1));
@@ -158,7 +198,7 @@ public int ls(String [] args) {
   }
 
   @SuppressWarnings("unchecked")
-  public int resolve(String [] args) {
+  public int resolve(String[] args) {
     Options resolveOption = new Options();
     CommandLineParser parser = new GnuParser();
     try {
@@ -166,10 +206,12 @@ public int resolve(String [] args) {
 
       List<String> argsList = line.getArgList();
       if (argsList.size() != 2) {
-				return usageError("resolve requires exactly one path argument", RESOLVE_USAGE);
+        return usageError("resolve requires exactly one path argument",
+            RESOLVE_USAGE);
       }
-			if (!validatePath(argsList.get(1)))
+      if (!validatePath(argsList.get(1))) {
         return -1;
+      }
 
       try {
         ServiceRecord record = registry.resolve(argsList.get(1));
@@ -182,12 +224,13 @@ public int resolve(String [] args) {
                          + endpoint.addressType + ") are: ");
 
           for (Map<String, String> address : endpoint.addresses) {
-            sysout.println("  [ ");
+            sysout.println("[ ");
             for (Map.Entry<String, String> entry : address.entrySet()) {
-              sysout.println("    " + entry.getKey()
+              sysout.print("\t" + entry.getKey()
-                             + ": \"" + entry.getValue() + "\"");
+                             + ":" + entry.getValue());
             }
-            sysout.println("  ]");
+
+            sysout.println("\n]");
           }
           sysout.println();
         }
@@ -196,13 +239,13 @@ public int resolve(String [] args) {
         syserr.println(analyzeException("resolve", e, argsList));
       }
       return -1;
-		} catch (org.apache.commons.cli.ParseException exp) {
+    } catch (ParseException exp) {
       return usageError("Invalid syntax " + exp, RESOLVE_USAGE);
     }
 
   }
 
-	public int bind(String [] args) {
+  public int bind(String[] args) {
     Option rest = OptionBuilder.withArgName("rest")
                                .hasArg()
                                .withDescription("rest Option")
@@ -243,7 +286,7 @@ public int bind(String [] args) {
 
     CommandLineParser parser = new GnuParser();
     ServiceRecord sr = new ServiceRecord();
-    CommandLine line = null;
+    CommandLine line;
     if (args.length <= 1) {
       return usageError("Invalid syntax ", BIND_USAGE);
     }
@@ -259,7 +302,12 @@ public int bind(String [] args) {
       }
       if (line.hasOption("inet") && line.hasOption("p") &&
           line.hasOption("h") && line.hasOption("api")) {
+        try {
           portNum = Integer.parseInt(line.getOptionValue("p"));
+        } catch (NumberFormatException exp) {
+          return usageError("Invalid Port - int required" + exp.getMessage(),
+              BIND_USAGE);
+        }
         hostName = line.getOptionValue("h");
         api = line.getOptionValue("api");
         sr.addExternalEndpoint(
@@ -278,7 +326,7 @@ public int bind(String [] args) {
         return usageError("Invalid syntax " + exp.getMessage(), BIND_USAGE);
       }
       if (line.hasOption("webui") && line.hasOption("api")) {
-        URI theUri = null;
+        URI theUri;
         try {
           theUri = new URI(line.getOptionValue("webui"));
         } catch (URISyntaxException e) {
@@ -319,8 +367,9 @@ public int bind(String [] args) {
     if (argsList.size() != 2) {
       return usageError("bind requires exactly one path argument", BIND_USAGE);
     }
-		if (!validatePath(argsList.get(1)))
+    if (!validatePath(argsList.get(1))) {
       return -1;
+    }
 
     try {
       registry.bind(argsList.get(1), sr, BindFlags.OVERWRITE);
@@ -333,7 +382,7 @@ public int bind(String [] args) {
   }
 
   @SuppressWarnings("unchecked")
-	public int mknode(String [] args) {
+  public int mknode(String[] args) {
     Options mknodeOption = new Options();
     CommandLineParser parser = new GnuParser();
     try {
@@ -341,10 +390,12 @@ public int mknode(String [] args) {
 
       List<String> argsList = line.getArgList();
       if (argsList.size() != 2) {
-				return usageError("mknode requires exactly one path argument", MKNODE_USAGE);
+        return usageError("mknode requires exactly one path argument",
+            MKNODE_USAGE);
       }
-			if (!validatePath(argsList.get(1)))
+      if (!validatePath(argsList.get(1))) {
         return -1;
+      }
 
       try {
         registry.mknode(args[1], false);
@@ -362,7 +413,8 @@ public int mknode(String [] args) {
   @SuppressWarnings("unchecked")
   public int rm(String[] args) {
     Option recursive = OptionBuilder.withArgName("recursive")
-				.withDescription("delete recursively").create("r");
+                                    .withDescription("delete recursively")
+                                    .create("r");
 
     Options rmOption = new Options();
     rmOption.addOption(recursive);
@@ -377,8 +429,9 @@ public int rm(String[] args) {
       if (argsList.size() != 2) {
         return usageError("RM requires exactly one path argument", RM_USAGE);
       }
-			if (!validatePath(argsList.get(1)))
+      if (!validatePath(argsList.get(1))) {
         return -1;
+      }
 
       try {
         if (line.hasOption("r")) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/cli/TestRegistryCli.java

@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.registry.cli;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+
+import org.apache.hadoop.registry.AbstractRegistryTest;
+import org.apache.hadoop.registry.operations.TestRegistryOperations;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestRegistryCli extends AbstractRegistryTest {
+  protected static final Logger LOG =
+      LoggerFactory.getLogger(TestRegistryOperations.class);
+
+  private ByteArrayOutputStream sysOutStream;
+  private PrintStream sysOut;
+  private ByteArrayOutputStream sysErrStream;
+  private PrintStream sysErr;
+  private RegistryCli cli;
+
+  @Before
+  public void setUp() throws Exception {
+    sysOutStream = new ByteArrayOutputStream();
+    sysOut = new PrintStream(sysOutStream);
+    sysErrStream = new ByteArrayOutputStream();
+    sysErr = new PrintStream(sysErrStream);
+    System.setOut(sysOut);
+    cli = new RegistryCli(operations, createRegistryConfiguration(), sysOut, sysErr);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    cli.close();
+  }
+
+  private void assertResult(RegistryCli cli, int code, String...args) throws Exception {
+    int result = cli.run(args);
+    assertEquals(code, result);
+  }
+
+  @Test
+  public void testBadCommands() throws Exception {
+    assertResult(cli, -1, new String[] { });
+    assertResult(cli, -1, "foo");
+  }
+
+  @Test
+  public void testInvalidNumArgs() throws Exception {
+    assertResult(cli, -1, "ls");
+    assertResult(cli, -1, "ls", "/path", "/extraPath");
+    assertResult(cli, -1, "resolve");
+    assertResult(cli, -1, "resolve", "/path", "/extraPath");
+    assertResult(cli, -1, "mknode");
+    assertResult(cli, -1, "mknode", "/path", "/extraPath");
+    assertResult(cli, -1, "rm");
+    assertResult(cli, -1, "rm", "/path", "/extraPath");
+    assertResult(cli, -1, "bind");
+    assertResult(cli, -1, "bind", "foo");
+    assertResult(cli, -1, "bind", "-inet", "foo");
+    assertResult(cli, -1, "bind", "-inet", "-api", "-p", "378", "-h", "host", "/foo");
+    assertResult(cli, -1, "bind", "-inet", "-api", "Api", "-p", "-h", "host", "/foo");
+    assertResult(cli, -1, "bind", "-inet", "-api", "Api", "-p", "378", "-h", "/foo");
+    assertResult(cli, -1, "bind", "-inet", "-api", "Api", "-p", "378", "-h", "host");
+    assertResult(cli, -1, "bind", "-api", "Api", "-p", "378", "-h", "host", "/foo");
+    assertResult(cli, -1, "bind", "-webui", "foo");
+    assertResult(cli, -1, "bind", "-webui", "-api", "Api", "/foo");
+    assertResult(cli, -1, "bind", "-webui", "uriString", "-api", "/foo");
+    assertResult(cli, -1, "bind", "-webui", "uriString", "-api", "Api");
+    assertResult(cli, -1, "bind", "-rest", "foo");
+    assertResult(cli, -1, "bind", "-rest", "uriString", "-api", "Api");
+    assertResult(cli, -1, "bind", "-rest", "-api", "Api", "/foo");
+    assertResult(cli, -1, "bind", "-rest", "uriString", "-api", "/foo");
+    assertResult(cli, -1, "bind", "uriString", "-api", "Api", "/foo");
+  }
+
+  @Test
+  public void testBadArgType() throws Exception {
+    assertResult(cli, -1, "bind", "-inet", "-api", "Api", "-p", "fooPort", "-h",
+        "host", "/dir");
+  }
+
+  @Test
+  public void testBadPath() throws Exception {
+    assertResult(cli, -1, "ls", "NonSlashPath");
+    assertResult(cli, -1, "ls", "//");
+    assertResult(cli, -1, "resolve", "NonSlashPath");
+    assertResult(cli, -1, "resolve", "//");
+    assertResult(cli, -1, "mknode", "NonSlashPath");
+    assertResult(cli, -1, "mknode", "//");
+    assertResult(cli, -1, "rm", "NonSlashPath");
+    assertResult(cli, -1, "rm", "//");
+    assertResult(cli, -1, "bind", "-inet", "-api", "Api", "-p", "378", "-h", "host", "NonSlashPath");
+    assertResult(cli, -1, "bind", "-inet", "-api", "Api", "-p", "378", "-h", "host", "//");
+    assertResult(cli, -1, "bind", "-webui", "uriString", "-api", "Api", "NonSlashPath");
+    assertResult(cli, -1, "bind", "-webui", "uriString", "-api", "Api", "//");
+    assertResult(cli, -1, "bind", "-rest", "uriString", "-api", "Api", "NonSlashPath");
+    assertResult(cli, -1, "bind", "-rest", "uriString", "-api", "Api", "//");
+  }
+
+  @Test
+  public void testNotExistingPaths() throws Exception {
+    assertResult(cli, -1, "ls", "/nonexisting_path");
+    assertResult(cli, -1, "ls", "/NonExistingDir/nonexisting_path");
+    assertResult(cli, -1, "resolve", "/nonexisting_path");
+    assertResult(cli, -1, "resolve", "/NonExistingDir/nonexisting_path");
+    assertResult(cli, -1, "bind", "-inet", "-api", "Api", "-p", "378", "-h", "host", "/NonExistingDir/nonexisting_path");
+    assertResult(cli, -1, "bind", "-webui", "uriString", "-api", "Api", "/NonExistingDir/nonexisting_path");
+    assertResult(cli, -1, "bind", "-rest", "uriString", "-api", "Api", "/NonExistingDir/nonexisting_path");
+  }
+
+  @Test
+  public void testValidCommands() throws Exception {
+    assertResult(cli, 0, "bind", "-inet", "-api", "Api", "-p", "378", "-h", "host", "/foo");
+    assertResult(cli, 0, "resolve", "/foo");
+    assertResult(cli, 0, "rm", "/foo");
+    assertResult(cli, -1, "resolve", "/foo");
+
+    assertResult(cli, 0, "bind", "-webui", "uriString", "-api", "Api", "/foo");
+    assertResult(cli, 0, "resolve", "/foo");
+    assertResult(cli, 0, "rm", "/foo");
+    assertResult(cli, -1, "resolve", "/foo");
+
+    assertResult(cli, 0, "bind", "-rest", "uriString", "-api", "Api", "/foo");
+    assertResult(cli, 0, "resolve", "/foo");
+    assertResult(cli, 0, "rm", "/foo");
+    assertResult(cli, -1, "resolve", "/foo");
+
+    //Test Sub Directories Binds
+    assertResult(cli, 0, "mknode", "/subdir");
+    assertResult(cli, -1, "resolve", "/subdir");
+
+    assertResult(cli, 0, "bind", "-inet", "-api", "Api", "-p", "378", "-h", "host", "/subdir/foo");
+    assertResult(cli, 0, "resolve", "/subdir/foo");
+    assertResult(cli, 0, "rm", "/subdir/foo");
+    assertResult(cli, -1, "resolve", "/subdir/foo");
+
+    assertResult(cli, 0, "bind", "-webui", "uriString", "-api", "Api", "/subdir/foo");
+    assertResult(cli, 0, "resolve", "/subdir/foo");
+    assertResult(cli, 0, "rm", "/subdir/foo");
+    assertResult(cli, -1, "resolve", "/subdir/foo");
+
+    assertResult(cli, 0, "bind", "-rest", "uriString", "-api", "Api", "/subdir/foo");
+    assertResult(cli, 0, "resolve", "/subdir/foo");
+    assertResult(cli, 0, "rm", "/subdir/foo");
+    assertResult(cli, -1, "resolve", "/subdir/foo");
+
+    assertResult(cli, 0, "rm", "/subdir");
+    assertResult(cli, -1, "resolve", "/subdir");
+
+    //Test Bind that the dir itself
+    assertResult(cli, 0, "mknode", "/dir");
+    assertResult(cli, -1, "resolve", "/dir");
+
+    assertResult(cli, 0, "bind", "-inet", "-api", "Api", "-p", "378", "-h", "host", "/dir");
+    assertResult(cli, 0, "resolve", "/dir");
+    assertResult(cli, 0, "rm", "/dir");
+    assertResult(cli, -1, "resolve", "/dir");
+
+    assertResult(cli, 0, "mknode", "/dir");
+    assertResult(cli, -1, "resolve", "/dir");
+
+    assertResult(cli, 0, "bind", "-webui", "uriString", "-api", "Api", "/dir");
+    assertResult(cli, 0, "resolve", "/dir");
+    assertResult(cli, 0, "rm", "/dir");
+    assertResult(cli, -1, "resolve", "/dir");
+
+    assertResult(cli, 0, "mknode", "/dir");
+    assertResult(cli, -1, "resolve", "/dir");
+
+    assertResult(cli, 0, "bind", "-rest", "uriString", "-api", "Api", "/dir");
+    assertResult(cli, 0, "resolve", "/dir");
+    assertResult(cli, 0, "rm", "/dir");
+    assertResult(cli, -1, "resolve", "/dir");
+
+    assertResult(cli, 0, "rm", "/Nonexitent");
+  }
+}