Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-8386 deprecate TableMapReduce.addDependencyJars(Configuration, class<?> ...) 

Signed-off-by: Matteo Bertozzi <matteo.bertozzi@cloudera.com>
This commit is contained in:
Sean Busbey 2016-07-19 07:21:26 -05:00
parent c59f76485e
commit 431c8c9ad0
12 changed files with 45 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java
View File

@ -785,7 +785,8 @@ public class IntegrationTestBigLinkedList extends IntegrationTestBase {
job.getConfiguration().setBoolean("mapreduce.map.speculative", false); job.getConfiguration().setBoolean("mapreduce.map.speculative", false);
TableMapReduceUtil.addDependencyJars(job); TableMapReduceUtil.addDependencyJars(job);
TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class); TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
AbstractHBaseTool.class);
TableMapReduceUtil.initCredentials(job); TableMapReduceUtil.initCredentials(job);
boolean success = jobCompletion(job); boolean success = jobCompletion(job);
@ -1296,7 +1297,8 @@ public class IntegrationTestBigLinkedList extends IntegrationTestBase {
TableMapReduceUtil.initTableMapperJob(getTableName(getConf()).getName(), scan, TableMapReduceUtil.initTableMapperJob(getTableName(getConf()).getName(), scan,
VerifyMapper.class, BytesWritable.class, BytesWritable.class, job); VerifyMapper.class, BytesWritable.class, BytesWritable.class, job);
TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class); TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
AbstractHBaseTool.class);
job.getConfiguration().setBoolean("mapreduce.map.speculative", false); job.getConfiguration().setBoolean("mapreduce.map.speculative", false);

4
hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java
View File

@ -348,7 +348,7 @@ public void cleanUpCluster() throws Exception {
TableMapReduceUtil.addDependencyJars(job); TableMapReduceUtil.addDependencyJars(job);
TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class); TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), AbstractHBaseTool.class);
TableMapReduceUtil.initCredentials(job); TableMapReduceUtil.initCredentials(job);
assertTrue(job.waitForCompletion(true)); assertTrue(job.waitForCompletion(true));
return job; return job;
@ -372,7 +372,7 @@ public void cleanUpCluster() throws Exception {
TableMapReduceUtil.initTableMapperJob( TableMapReduceUtil.initTableMapperJob(
htd.getTableName().getNameAsString(), scan, VerifyMapper.class, htd.getTableName().getNameAsString(), scan, VerifyMapper.class,
BytesWritable.class, BytesWritable.class, job); BytesWritable.class, BytesWritable.class, job);
TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class); TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), AbstractHBaseTool.class);
int scannerCaching = conf.getInt("verify.scannercaching", SCANNER_CACHING); int scannerCaching = conf.getInt("verify.scannercaching", SCANNER_CACHING);
TableMapReduceUtil.setScannerCaching(job, scannerCaching); TableMapReduceUtil.setScannerCaching(job, scannerCaching);

2
hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestWithCellVisibilityLoadAndVerify.java
View File

@ -331,7 +331,7 @@ public class IntegrationTestWithCellVisibilityLoadAndVerify extends IntegrationT
scan.setAuthorizations(new Authorizations(auths)); scan.setAuthorizations(new Authorizations(auths));
TableMapReduceUtil.initTableMapperJob(htd.getTableName().getNameAsString(), scan, TableMapReduceUtil.initTableMapperJob(htd.getTableName().getNameAsString(), scan,
VerifyMapper.class, NullWritable.class, NullWritable.class, job); VerifyMapper.class, NullWritable.class, NullWritable.class, job);
TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class); TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), AbstractHBaseTool.class);
int scannerCaching = conf.getInt("verify.scannercaching", SCANNER_CACHING); int scannerCaching = conf.getInt("verify.scannercaching", SCANNER_CACHING);
TableMapReduceUtil.setScannerCaching(job, scannerCaching); TableMapReduceUtil.setScannerCaching(job, scannerCaching);
job.setNumReduceTasks(0); job.setNumReduceTasks(0);

2
hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
View File

@ -362,7 +362,7 @@ public class TableMapReduceUtil {
*/ */
public static void addDependencyJars(JobConf job) throws IOException { public static void addDependencyJars(JobConf job) throws IOException {
org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addHBaseDependencyJars(job); org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addHBaseDependencyJars(job);
org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addDependencyJars( org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addDependencyJarsForClasses(
job, job,
// when making changes here, consider also mapreduce.TableMapReduceUtil // when making changes here, consider also mapreduce.TableMapReduceUtil
// pull job classes // pull job classes

6
hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
View File

@ -616,7 +616,7 @@ public class Import extends Configured implements Tool {
try { try {
Class<? extends Filter> filter = conf.getClass(FILTER_CLASS_CONF_KEY, null, Filter.class); Class<? extends Filter> filter = conf.getClass(FILTER_CLASS_CONF_KEY, null, Filter.class);
if (filter != null) { if (filter != null) {
TableMapReduceUtil.addDependencyJars(conf, filter); TableMapReduceUtil.addDependencyJarsForClasses(conf, filter);
} }
} catch (Exception e) { } catch (Exception e) {
throw new IOException(e); throw new IOException(e);
@ -643,7 +643,7 @@ public class Import extends Configured implements Tool {
fs.deleteOnExit(partitionsPath); fs.deleteOnExit(partitionsPath);
job.setPartitionerClass(KeyValueWritableComparablePartitioner.class); job.setPartitionerClass(KeyValueWritableComparablePartitioner.class);
job.setNumReduceTasks(regionLocator.getStartKeys().length); job.setNumReduceTasks(regionLocator.getStartKeys().length);
TableMapReduceUtil.addDependencyJars(job.getConfiguration(), TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
com.google.common.base.Preconditions.class); com.google.common.base.Preconditions.class);
} }
} else if (hfileOutPath != null) { } else if (hfileOutPath != null) {
@ -658,7 +658,7 @@ public class Import extends Configured implements Tool {
job.setMapOutputKeyClass(ImmutableBytesWritable.class); job.setMapOutputKeyClass(ImmutableBytesWritable.class);
job.setMapOutputValueClass(KeyValue.class); job.setMapOutputValueClass(KeyValue.class);
HFileOutputFormat2.configureIncrementalLoad(job, table.getTableDescriptor(), regionLocator); HFileOutputFormat2.configureIncrementalLoad(job, table.getTableDescriptor(), regionLocator);
TableMapReduceUtil.addDependencyJars(job.getConfiguration(), TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
com.google.common.base.Preconditions.class); com.google.common.base.Preconditions.class);
} }
} else { } else {

2
hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
View File

@ -587,7 +587,7 @@ public class ImportTsv extends Configured implements Tool {
KeyValueSerialization.class.getName()); KeyValueSerialization.class.getName());
} }
TableMapReduceUtil.addDependencyJars(job); TableMapReduceUtil.addDependencyJars(job);
TableMapReduceUtil.addDependencyJars(job.getConfiguration(), TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
com.google.common.base.Function.class /* Guava used by TsvParser */); com.google.common.base.Function.class /* Guava used by TsvParser */);
} }
} }

30
hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
View File

@ -339,7 +339,7 @@ public class TableMapReduceUtil {
if (addDependencyJars) { if (addDependencyJars) {
addDependencyJars(job); addDependencyJars(job);
addDependencyJars(job.getConfiguration(), MetricRegistry.class); addDependencyJarsForClasses(job.getConfiguration(), MetricRegistry.class);
} }
resetCacheConfig(job.getConfiguration()); resetCacheConfig(job.getConfiguration());
@ -787,7 +787,7 @@ public class TableMapReduceUtil {
" Continuing without it."); " Continuing without it.");
} }
addDependencyJars(conf, addDependencyJarsForClasses(conf,
// explicitly pull a class from each module // explicitly pull a class from each module
org.apache.hadoop.hbase.HConstants.class, // hbase-common org.apache.hadoop.hbase.HConstants.class, // hbase-common
org.apache.hadoop.hbase.protobuf.generated.ClientProtos.class, // hbase-protocol org.apache.hadoop.hbase.protobuf.generated.ClientProtos.class, // hbase-protocol
@ -835,7 +835,7 @@ public class TableMapReduceUtil {
public static void addDependencyJars(Job job) throws IOException { public static void addDependencyJars(Job job) throws IOException {
addHBaseDependencyJars(job.getConfiguration()); addHBaseDependencyJars(job.getConfiguration());
try { try {
addDependencyJars(job.getConfiguration(), addDependencyJarsForClasses(job.getConfiguration(),
// when making changes here, consider also mapred.TableMapReduceUtil // when making changes here, consider also mapred.TableMapReduceUtil
// pull job classes // pull job classes
job.getMapOutputKeyClass(), job.getMapOutputKeyClass(),
@ -855,9 +855,33 @@ public class TableMapReduceUtil {
* Add the jars containing the given classes to the job's configuration * Add the jars containing the given classes to the job's configuration
* such that JobClient will ship them to the cluster and add them to * such that JobClient will ship them to the cluster and add them to
* the DistributedCache. * the DistributedCache.
* @deprecated rely on {@link #addDependencyJars(Job)} instead.
*/ */
@Deprecated
public static void addDependencyJars(Configuration conf, public static void addDependencyJars(Configuration conf,
Class<?>... classes) throws IOException { Class<?>... classes) throws IOException {
LOG.warn("The addDependencyJars(Configuration, Class<?>...) method has been deprecated since it"
+ " is easy to use incorrectly. Most users should rely on addDependencyJars(Job) " +
"instead. See HBASE-8386 for more details.");
addDependencyJarsForClasses(conf, classes);
}
/**
* Add the jars containing the given classes to the job's configuration
* such that JobClient will ship them to the cluster and add them to
* the DistributedCache.
*
* N.B. that this method at most adds one jar per class given. If there is more than one
* jar available containing a class with the same name as a given class, we don't define
* which of those jars might be chosen.
*
* @param conf The Hadoop Configuration to modify
* @param classes will add just those dependencies needed to find the given classes
* @throws IOException if an underlying library call fails.
*/
@InterfaceAudience.Private
public static void addDependencyJarsForClasses(Configuration conf,
Class<?>... classes) throws IOException {
FileSystem localFs = FileSystem.getLocal(conf); FileSystem localFs = FileSystem.getLocal(conf);
Set<String> jars = new HashSet<String>(); Set<String> jars = new HashSet<String>();

4
hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
View File

@ -282,7 +282,7 @@ public class WALPlayer extends Configured implements Tool {
RegionLocator regionLocator = conn.getRegionLocator(tableName)) { RegionLocator regionLocator = conn.getRegionLocator(tableName)) {
HFileOutputFormat2.configureIncrementalLoad(job, table.getTableDescriptor(), regionLocator); HFileOutputFormat2.configureIncrementalLoad(job, table.getTableDescriptor(), regionLocator);
} }
TableMapReduceUtil.addDependencyJars(job.getConfiguration(), TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
com.google.common.base.Preconditions.class); com.google.common.base.Preconditions.class);
} else { } else {
// output to live cluster // output to live cluster
@ -295,7 +295,7 @@ public class WALPlayer extends Configured implements Tool {
} }
String codecCls = WALCellCodec.getWALCellCodecClass(conf); String codecCls = WALCellCodec.getWALCellCodecClass(conf);
try { try {
TableMapReduceUtil.addDependencyJars(job.getConfiguration(), Class.forName(codecCls)); TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), Class.forName(codecCls));
} catch (Exception e) { } catch (Exception e) {
throw new IOException("Cannot determine wal codec class " + codecCls, e); throw new IOException("Cannot determine wal codec class " + codecCls, e);
} }

2
hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
View File

@ -499,7 +499,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
TextOutputFormat.setOutputPath(job, new Path(inputDir.getParent(), "outputs")); TextOutputFormat.setOutputPath(job, new Path(inputDir.getParent(), "outputs"));
TableMapReduceUtil.addDependencyJars(job); TableMapReduceUtil.addDependencyJars(job);
TableMapReduceUtil.addDependencyJars(job.getConfiguration(), TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
Histogram.class, // yammer metrics Histogram.class, // yammer metrics
ObjectMapper.class); // jackson-mapper-asl ObjectMapper.class); // jackson-mapper-asl

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableSnapshotInputFormat.java
View File

@ -243,7 +243,7 @@ public class TestTableSnapshotInputFormat extends TableSnapshotInputFormatTestBa
JobConf jobConf = new JobConf(util.getConfiguration()); JobConf jobConf = new JobConf(util.getConfiguration());
jobConf.setJarByClass(util.getClass()); jobConf.setJarByClass(util.getClass());
org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addDependencyJars(jobConf, org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addDependencyJarsForClasses(jobConf,
TestTableSnapshotInputFormat.class); TestTableSnapshotInputFormat.class);
TableMapReduceUtil.initTableSnapshotMapJob(snapshotName, COLUMNS, TableMapReduceUtil.initTableSnapshotMapJob(snapshotName, COLUMNS,

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java
View File

@ -281,7 +281,7 @@ public class TestTableSnapshotInputFormat extends TableSnapshotInputFormatTestBa
Scan scan = new Scan(startRow, endRow); // limit the scan Scan scan = new Scan(startRow, endRow); // limit the scan
job.setJarByClass(util.getClass()); job.setJarByClass(util.getClass());
TableMapReduceUtil.addDependencyJars(job.getConfiguration(), TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
TestTableSnapshotInputFormat.class); TestTableSnapshotInputFormat.class);
TableMapReduceUtil.initTableSnapshotMapperJob(snapshotName, TableMapReduceUtil.initTableSnapshotMapperJob(snapshotName,

2
src/main/asciidoc/_chapters/mapreduce.adoc
View File

@ -73,7 +73,7 @@ This example assumes you use a BASH-compatible shell.
$ HADOOP_CLASSPATH=`${HBASE_HOME}/bin/hbase classpath` ${HADOOP_HOME}/bin/hadoop jar ${HBASE_HOME}/lib/hbase-server-VERSION.jar rowcounter usertable $ HADOOP_CLASSPATH=`${HBASE_HOME}/bin/hbase classpath` ${HADOOP_HOME}/bin/hadoop jar ${HBASE_HOME}/lib/hbase-server-VERSION.jar rowcounter usertable
---- ----
When the command runs, internally, the HBase JAR finds the dependencies it needs for ZooKeeper, Guava, and its other dependencies on the passed `HADOOP_CLASSPATH` and adds the JARs to the MapReduce job configuration. When the command runs, internally, the HBase JAR finds the dependencies it needs and adds them to the MapReduce job configuration.
See the source at `TableMapReduceUtil#addDependencyJars(org.apache.hadoop.mapreduce.Job)` for how this is done. See the source at `TableMapReduceUtil#addDependencyJars(org.apache.hadoop.mapreduce.Job)` for how this is done.
The command `hbase mapredcp` can also help you dump the CLASSPATH entries required by MapReduce, which are the same jars `TableMapReduceUtil#addDependencyJars` would add. The command `hbase mapredcp` can also help you dump the CLASSPATH entries required by MapReduce, which are the same jars `TableMapReduceUtil#addDependencyJars` would add.