HBASE-8438 Extend bin/hbase to print a "mapreduce classpath"

$ ./bin/hbase mapredcp --help
Usage: hbase mapredcp [-Dtmpjars=...]
  Construct a CLASSPATH containing dependency jars required to run a mapreduce
  job. By default, includes any jars detected by TableMapReduceUtils. Provide
  additional entries by specifying a comma-separated list in tmpjars.

$ ./bin/hbase mapredcp | tr ':' '\n'
/private/tmp/hbase-0.97.0-SNAPSHOT/lib/netty-3.6.6.Final.jar
/private/tmp/hbase-0.97.0-SNAPSHOT/lib/hbase-hadoop-compat-0.97.0-SNAPSHOT.jar
/private/tmp/hbase-0.97.0-SNAPSHOT/lib/protobuf-java-2.5.0.jar
/private/tmp/hbase-0.97.0-SNAPSHOT/lib/guava-12.0.1.jar
/private/tmp/hbase-0.97.0-SNAPSHOT/lib/htrace-core-2.01.jar
/private/tmp/hbase-0.97.0-SNAPSHOT/lib/hbase-protocol-0.97.0-SNAPSHOT.jar
/private/tmp/hbase-0.97.0-SNAPSHOT/lib/hbase-client-0.97.0-SNAPSHOT.jar
/private/tmp/hbase-0.97.0-SNAPSHOT/lib/zookeeper-3.4.5.jar
/private/tmp/hbase-0.97.0-SNAPSHOT/lib/hbase-server-0.97.0-SNAPSHOT.jar
/private/tmp/hbase-0.97.0-SNAPSHOT/lib/hbase-common-0.97.0-SNAPSHOT.jar


git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1542811 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
ndimiduk 2013-11-17 20:49:24 +00:00
parent b0574a9528
commit e1a3d71738
4 changed files with 103 additions and 5 deletions

View File

@ -92,6 +92,7 @@ if [ $# = 0 ]; then
echo " thrift2 Run the HBase Thrift2 server"
echo " clean Run the HBase clean up script"
echo " classpath Dump hbase CLASSPATH"
echo " mapredcp Dump CLASSPATH entries required by mapreduce"
echo " version Print the version"
echo " CLASSNAME Run the class named CLASSNAME"
exit 1
@ -324,7 +325,8 @@ elif [ "$COMMAND" = "clean" ] ; then
fi
"$bin"/hbase-cleanup.sh --config ${HBASE_CONF_DIR} $@
exit $?
elif [ "$COMMAND" = "mapredcp" ] ; then
CLASS='org.apache.hadoop.hbase.util.MapreduceDependencyClasspathTool'
elif [ "$COMMAND" = "classpath" ] ; then
echo $CLASSPATH
exit 0

View File

@ -202,7 +202,7 @@ goto :MakeCmdArgsLoop
set hbase-command-arguments=%_hbasearguments%
@rem figure out which class to run
set corecommands=shell master regionserver thrift thrift2 rest avro hlog hbck hfile zookeeper zkcli upgrade
set corecommands=shell master regionserver thrift thrift2 rest avro hlog hbck hfile zookeeper zkcli upgrade mapredcp
for %%i in ( %corecommands% ) do (
if "%hbase-command%"=="%%i" set corecommand=true
)
@ -380,6 +380,10 @@ goto :eof
set CLASS=org.apache.hadoop.hbase.migration.UpgradeTo96
goto :eof
:mapredcp
set CLASS=org.apache.hadoop.hbase.util.MapreduceDependencyClasspathTool
goto :eof
:makeServiceXml
set arguments=%*
@echo ^<service^>
@ -412,6 +416,7 @@ goto :eof
echo thrift Run the HBase Thrift server
echo thrift2 Run the HBase Thrift2 server
echo classpath Dump hbase CLASSPATH
echo mapredcp Dump CLASSPATH entries required by mapreduce
echo version Print the version
echo CLASSNAME Run the class named CLASSNAME
goto :eof

View File

@ -18,6 +18,7 @@
*/
package org.apache.hadoop.hbase.mapreduce;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
@ -99,7 +100,6 @@ public class TableMapReduceUtil {
job, true);
}
/**
* Use this before submitting a TableMap job. It will appropriately set up
* the job.
@ -585,6 +585,28 @@ public class TableMapReduceUtil {
org.cloudera.htrace.Trace.class);
}
/**
* Returns a classpath string built from the content of the "tmpjars" value in {@code conf}.
* Also exposed to shell scripts via `bin/hbase mapredcp`.
*/
public static String buildDependencyClasspath(Configuration conf) {
if (conf == null) {
throw new IllegalArgumentException("Must provide a configuration object.");
}
Set<String> paths = new HashSet<String>(conf.getStringCollection("tmpjars"));
if (paths.size() == 0) {
throw new IllegalArgumentException("Configuration contains no tmpjars.");
}
StringBuilder sb = new StringBuilder();
for (String s : paths) {
// entries can take the form 'file:/path/to/file.jar'.
int idx = s.indexOf(":");
if (idx != -1) s = s.substring(idx + 1);
if (sb.length() > 0) sb.append(File.pathSeparator);
sb.append(s);
}
return sb.toString();
}
/**
* Add the HBase dependency jars as well as jars for any of the configured
@ -646,8 +668,7 @@ public class TableMapReduceUtil {
}
if (jars.isEmpty()) return;
conf.set("tmpjars",
StringUtils.arrayToString(jars.toArray(new String[0])));
conf.set("tmpjars", StringUtils.arrayToString(jars.toArray(new String[jars.size()])));
}
/**

View File

@ -0,0 +1,70 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.util;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
/**
* Generate a classpath string containing any jars required by mapreduce jobs. Specify
* additional values by providing a comma-separated list of paths via -Dtmpjars.
*/
public class MapreduceDependencyClasspathTool implements Tool {
private Configuration conf;
@Override
public void setConf(Configuration conf) {
this.conf = conf;
}
@Override
public Configuration getConf() {
return conf;
}
@Override
public int run(String[] args) throws Exception {
if (args.length > 0) {
System.err.println("Usage: hbase mapredcp [-Dtmpjars=...]");
System.err.println(" Construct a CLASSPATH containing dependency jars required to run a mapreduce");
System.err.println(" job. By default, includes any jars detected by TableMapReduceUtils. Provide");
System.err.println(" additional entries by specifying a comma-separated list in tmpjars.");
return 0;
}
TableMapReduceUtil.addHBaseDependencyJars(getConf());
System.out.println(TableMapReduceUtil.buildDependencyClasspath(getConf()));
return 0;
}
public static void main(String[] argv) throws Exception {
// Silence the usual noise. This is probably fragile...
Logger logger = Logger.getLogger("org.apache.hadoop.hbase");
if (logger != null) {
logger.setLevel(Level.WARN);
}
System.exit(ToolRunner.run(
HBaseConfiguration.create(), new MapreduceDependencyClasspathTool(), argv));
}
}