HBASE-8438 Extend bin/hbase to print a "mapreduce classpath"
$ ./bin/hbase mapredcp --help Usage: hbase mapredcp [-Dtmpjars=...] Construct a CLASSPATH containing dependency jars required to run a mapreduce job. By default, includes any jars detected by TableMapReduceUtils. Provide additional entries by specifying a comma-separated list in tmpjars. $ ./bin/hbase mapredcp | tr ':' '\n' /private/tmp/hbase-0.97.0-SNAPSHOT/lib/netty-3.6.6.Final.jar /private/tmp/hbase-0.97.0-SNAPSHOT/lib/hbase-hadoop-compat-0.97.0-SNAPSHOT.jar /private/tmp/hbase-0.97.0-SNAPSHOT/lib/protobuf-java-2.5.0.jar /private/tmp/hbase-0.97.0-SNAPSHOT/lib/guava-12.0.1.jar /private/tmp/hbase-0.97.0-SNAPSHOT/lib/htrace-core-2.01.jar /private/tmp/hbase-0.97.0-SNAPSHOT/lib/hbase-protocol-0.97.0-SNAPSHOT.jar /private/tmp/hbase-0.97.0-SNAPSHOT/lib/hbase-client-0.97.0-SNAPSHOT.jar /private/tmp/hbase-0.97.0-SNAPSHOT/lib/zookeeper-3.4.5.jar /private/tmp/hbase-0.97.0-SNAPSHOT/lib/hbase-server-0.97.0-SNAPSHOT.jar /private/tmp/hbase-0.97.0-SNAPSHOT/lib/hbase-common-0.97.0-SNAPSHOT.jar git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1542811 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
b0574a9528
commit
e1a3d71738
|
@ -92,6 +92,7 @@ if [ $# = 0 ]; then
|
|||
echo " thrift2 Run the HBase Thrift2 server"
|
||||
echo " clean Run the HBase clean up script"
|
||||
echo " classpath Dump hbase CLASSPATH"
|
||||
echo " mapredcp Dump CLASSPATH entries required by mapreduce"
|
||||
echo " version Print the version"
|
||||
echo " CLASSNAME Run the class named CLASSNAME"
|
||||
exit 1
|
||||
|
@ -324,7 +325,8 @@ elif [ "$COMMAND" = "clean" ] ; then
|
|||
fi
|
||||
"$bin"/hbase-cleanup.sh --config ${HBASE_CONF_DIR} $@
|
||||
exit $?
|
||||
|
||||
elif [ "$COMMAND" = "mapredcp" ] ; then
|
||||
CLASS='org.apache.hadoop.hbase.util.MapreduceDependencyClasspathTool'
|
||||
elif [ "$COMMAND" = "classpath" ] ; then
|
||||
echo $CLASSPATH
|
||||
exit 0
|
||||
|
|
|
@ -202,7 +202,7 @@ goto :MakeCmdArgsLoop
|
|||
set hbase-command-arguments=%_hbasearguments%
|
||||
|
||||
@rem figure out which class to run
|
||||
set corecommands=shell master regionserver thrift thrift2 rest avro hlog hbck hfile zookeeper zkcli upgrade
|
||||
set corecommands=shell master regionserver thrift thrift2 rest avro hlog hbck hfile zookeeper zkcli upgrade mapredcp
|
||||
for %%i in ( %corecommands% ) do (
|
||||
if "%hbase-command%"=="%%i" set corecommand=true
|
||||
)
|
||||
|
@ -380,6 +380,10 @@ goto :eof
|
|||
set CLASS=org.apache.hadoop.hbase.migration.UpgradeTo96
|
||||
goto :eof
|
||||
|
||||
:mapredcp
|
||||
set CLASS=org.apache.hadoop.hbase.util.MapreduceDependencyClasspathTool
|
||||
goto :eof
|
||||
|
||||
:makeServiceXml
|
||||
set arguments=%*
|
||||
@echo ^<service^>
|
||||
|
@ -412,6 +416,7 @@ goto :eof
|
|||
echo thrift Run the HBase Thrift server
|
||||
echo thrift2 Run the HBase Thrift2 server
|
||||
echo classpath Dump hbase CLASSPATH
|
||||
echo mapredcp Dump CLASSPATH entries required by mapreduce
|
||||
echo version Print the version
|
||||
echo CLASSNAME Run the class named CLASSNAME
|
||||
goto :eof
|
||||
|
|
|
@ -18,6 +18,7 @@
|
|||
*/
|
||||
package org.apache.hadoop.hbase.mapreduce;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.lang.reflect.InvocationTargetException;
|
||||
import java.lang.reflect.Method;
|
||||
|
@ -99,7 +100,6 @@ public class TableMapReduceUtil {
|
|||
job, true);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Use this before submitting a TableMap job. It will appropriately set up
|
||||
* the job.
|
||||
|
@ -585,6 +585,28 @@ public class TableMapReduceUtil {
|
|||
org.cloudera.htrace.Trace.class);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a classpath string built from the content of the "tmpjars" value in {@code conf}.
|
||||
* Also exposed to shell scripts via `bin/hbase mapredcp`.
|
||||
*/
|
||||
public static String buildDependencyClasspath(Configuration conf) {
|
||||
if (conf == null) {
|
||||
throw new IllegalArgumentException("Must provide a configuration object.");
|
||||
}
|
||||
Set<String> paths = new HashSet<String>(conf.getStringCollection("tmpjars"));
|
||||
if (paths.size() == 0) {
|
||||
throw new IllegalArgumentException("Configuration contains no tmpjars.");
|
||||
}
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (String s : paths) {
|
||||
// entries can take the form 'file:/path/to/file.jar'.
|
||||
int idx = s.indexOf(":");
|
||||
if (idx != -1) s = s.substring(idx + 1);
|
||||
if (sb.length() > 0) sb.append(File.pathSeparator);
|
||||
sb.append(s);
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Add the HBase dependency jars as well as jars for any of the configured
|
||||
|
@ -646,8 +668,7 @@ public class TableMapReduceUtil {
|
|||
}
|
||||
if (jars.isEmpty()) return;
|
||||
|
||||
conf.set("tmpjars",
|
||||
StringUtils.arrayToString(jars.toArray(new String[0])));
|
||||
conf.set("tmpjars", StringUtils.arrayToString(jars.toArray(new String[jars.size()])));
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
@ -0,0 +1,70 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.util;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.hbase.HBaseConfiguration;
|
||||
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
|
||||
import org.apache.hadoop.util.Tool;
|
||||
import org.apache.hadoop.util.ToolRunner;
|
||||
import org.apache.log4j.Level;
|
||||
import org.apache.log4j.Logger;
|
||||
|
||||
/**
|
||||
* Generate a classpath string containing any jars required by mapreduce jobs. Specify
|
||||
* additional values by providing a comma-separated list of paths via -Dtmpjars.
|
||||
*/
|
||||
public class MapreduceDependencyClasspathTool implements Tool {
|
||||
|
||||
private Configuration conf;
|
||||
|
||||
@Override
|
||||
public void setConf(Configuration conf) {
|
||||
this.conf = conf;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Configuration getConf() {
|
||||
return conf;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int run(String[] args) throws Exception {
|
||||
if (args.length > 0) {
|
||||
System.err.println("Usage: hbase mapredcp [-Dtmpjars=...]");
|
||||
System.err.println(" Construct a CLASSPATH containing dependency jars required to run a mapreduce");
|
||||
System.err.println(" job. By default, includes any jars detected by TableMapReduceUtils. Provide");
|
||||
System.err.println(" additional entries by specifying a comma-separated list in tmpjars.");
|
||||
return 0;
|
||||
}
|
||||
|
||||
TableMapReduceUtil.addHBaseDependencyJars(getConf());
|
||||
System.out.println(TableMapReduceUtil.buildDependencyClasspath(getConf()));
|
||||
return 0;
|
||||
}
|
||||
|
||||
public static void main(String[] argv) throws Exception {
|
||||
// Silence the usual noise. This is probably fragile...
|
||||
Logger logger = Logger.getLogger("org.apache.hadoop.hbase");
|
||||
if (logger != null) {
|
||||
logger.setLevel(Level.WARN);
|
||||
}
|
||||
System.exit(ToolRunner.run(
|
||||
HBaseConfiguration.create(), new MapreduceDependencyClasspathTool(), argv));
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue