HBASE-8438 Extend bin/hbase to print a "mapreduce classpath"

$ ./bin/hbase mapredcp --help
Usage: hbase mapredcp [-Dtmpjars=...]
  Construct a CLASSPATH containing dependency jars required to run a mapreduce
  job. By default, includes any jars detected by TableMapReduceUtils. Provide
  additional entries by specifying a comma-separated list in tmpjars.

$ ./bin/hbase mapredcp | tr ':' '\n'
/private/tmp/hbase-0.97.0-SNAPSHOT/lib/netty-3.6.6.Final.jar
/private/tmp/hbase-0.97.0-SNAPSHOT/lib/hbase-hadoop-compat-0.97.0-SNAPSHOT.jar
/private/tmp/hbase-0.97.0-SNAPSHOT/lib/protobuf-java-2.5.0.jar
/private/tmp/hbase-0.97.0-SNAPSHOT/lib/guava-12.0.1.jar
/private/tmp/hbase-0.97.0-SNAPSHOT/lib/htrace-core-2.01.jar
/private/tmp/hbase-0.97.0-SNAPSHOT/lib/hbase-protocol-0.97.0-SNAPSHOT.jar
/private/tmp/hbase-0.97.0-SNAPSHOT/lib/hbase-client-0.97.0-SNAPSHOT.jar
/private/tmp/hbase-0.97.0-SNAPSHOT/lib/zookeeper-3.4.5.jar
/private/tmp/hbase-0.97.0-SNAPSHOT/lib/hbase-server-0.97.0-SNAPSHOT.jar
/private/tmp/hbase-0.97.0-SNAPSHOT/lib/hbase-common-0.97.0-SNAPSHOT.jar


git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1542811 13f79535-47bb-0310-9956-ffa450edef68

bin/hbase

@@ -92,6 +92,7 @@ if [ $# = 0 ]; then
   echo "  thrift2         Run the HBase Thrift2 server"
   echo "  clean           Run the HBase clean up script"
   echo "  classpath       Dump hbase CLASSPATH"
+  echo "  mapredcp        Dump CLASSPATH entries required by mapreduce"
   echo "  version         Print the version"
   echo "  CLASSNAME       Run the class named CLASSNAME"
   exit 1
@@ -324,7 +325,8 @@ elif [ "$COMMAND" = "clean" ] ; then
   fi
   "$bin"/hbase-cleanup.sh --config ${HBASE_CONF_DIR} $@
   exit $?
-
+elif [ "$COMMAND" = "mapredcp" ] ; then
+  CLASS='org.apache.hadoop.hbase.util.MapreduceDependencyClasspathTool'
 elif [ "$COMMAND" = "classpath" ] ; then
   echo $CLASSPATH
   exit 0

bin/hbase.cmd

@@ -202,7 +202,7 @@ goto :MakeCmdArgsLoop
 set hbase-command-arguments=%_hbasearguments%
 
 @rem figure out which class to run
-set corecommands=shell master regionserver thrift thrift2 rest avro hlog hbck hfile zookeeper zkcli upgrade
+set corecommands=shell master regionserver thrift thrift2 rest avro hlog hbck hfile zookeeper zkcli upgrade mapredcp
 for %%i in ( %corecommands% ) do (
   if "%hbase-command%"=="%%i" set corecommand=true
 )
@@ -380,6 +380,10 @@ goto :eof
   set CLASS=org.apache.hadoop.hbase.migration.UpgradeTo96
   goto :eof
 
+:mapredcp
+  set CLASS=org.apache.hadoop.hbase.util.MapreduceDependencyClasspathTool
+  goto :eof
+
 :makeServiceXml
   set arguments=%*
   @echo ^<service^>
@@ -412,6 +416,7 @@ goto :eof
   echo   thrift          Run the HBase Thrift server
   echo   thrift2         Run the HBase Thrift2 server
   echo   classpath       Dump hbase CLASSPATH
+  echo   mapredcp        Dump CLASSPATH entries required by mapreduce
   echo   version         Print the version
   echo   CLASSNAME       Run the class named CLASSNAME
   goto :eof

hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java

@@ -18,6 +18,7 @@
  */
 package org.apache.hadoop.hbase.mapreduce;
 
+import java.io.File;
 import java.io.IOException;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
@@ -99,7 +100,6 @@ public class TableMapReduceUtil {
         job, true);
   }
 
-
   /**
    * Use this before submitting a TableMap job. It will appropriately set up
    * the job.
@@ -585,6 +585,28 @@ public class TableMapReduceUtil {
       org.cloudera.htrace.Trace.class);
   }
 
+  /**
+   * Returns a classpath string built from the content of the "tmpjars" value in {@code conf}.
+   * Also exposed to shell scripts via `bin/hbase mapredcp`.
+   */
+  public static String buildDependencyClasspath(Configuration conf) {
+    if (conf == null) {
+      throw new IllegalArgumentException("Must provide a configuration object.");
+    }
+    Set<String> paths = new HashSet<String>(conf.getStringCollection("tmpjars"));
+    if (paths.size() == 0) {
+      throw new IllegalArgumentException("Configuration contains no tmpjars.");
+    }
+    StringBuilder sb = new StringBuilder();
+    for (String s : paths) {
+      // entries can take the form 'file:/path/to/file.jar'.
+      int idx = s.indexOf(":");
+      if (idx != -1) s = s.substring(idx + 1);
+      if (sb.length() > 0) sb.append(File.pathSeparator);
+      sb.append(s);
+    }
+    return sb.toString();
+  }
 
   /**
    * Add the HBase dependency jars as well as jars for any of the configured
@@ -646,8 +668,7 @@ public class TableMapReduceUtil {
     }
     if (jars.isEmpty()) return;
 
-    conf.set("tmpjars",
+    conf.set("tmpjars", StringUtils.arrayToString(jars.toArray(new String[jars.size()])));
-             StringUtils.arrayToString(jars.toArray(new String[0])));
   }
 
   /**

hbase-server/src/main/java/org/apache/hadoop/hbase/util/MapreduceDependencyClasspathTool.java

@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * Generate a classpath string containing any jars required by mapreduce jobs. Specify
+ * additional values by providing a comma-separated list of paths via -Dtmpjars.
+ */
+public class MapreduceDependencyClasspathTool implements Tool {
+
+  private Configuration conf;
+
+  @Override
+  public void setConf(Configuration conf) {
+    this.conf = conf;
+  }
+
+  @Override
+  public Configuration getConf() {
+    return conf;
+  }
+
+  @Override
+  public int run(String[] args) throws Exception {
+    if (args.length > 0) {
+      System.err.println("Usage: hbase mapredcp [-Dtmpjars=...]");
+      System.err.println("  Construct a CLASSPATH containing dependency jars required to run a mapreduce");
+      System.err.println("  job. By default, includes any jars detected by TableMapReduceUtils. Provide");
+      System.err.println("  additional entries by specifying a comma-separated list in tmpjars.");
+      return 0;
+    }
+
+    TableMapReduceUtil.addHBaseDependencyJars(getConf());
+    System.out.println(TableMapReduceUtil.buildDependencyClasspath(getConf()));
+    return 0;
+  }
+
+  public static void main(String[] argv) throws Exception {
+    // Silence the usual noise. This is probably fragile...
+    Logger logger = Logger.getLogger("org.apache.hadoop.hbase");
+    if (logger != null) {
+      logger.setLevel(Level.WARN);
+    }
+    System.exit(ToolRunner.run(
+      HBaseConfiguration.create(), new MapreduceDependencyClasspathTool(), argv));
+  }
+}