From 431c8c9ad03336b5ab844d97adc47b8a73d7aa00 Mon Sep 17 00:00:00 2001 From: Sean Busbey Date: Tue, 19 Jul 2016 07:21:26 -0500 Subject: [PATCH] HBASE-8386 deprecate TableMapReduce.addDependencyJars(Configuration, class ...) Signed-off-by: Matteo Bertozzi --- .../test/IntegrationTestBigLinkedList.java | 6 ++-- .../test/IntegrationTestLoadAndVerify.java | 4 +-- ...onTestWithCellVisibilityLoadAndVerify.java | 2 +- .../hbase/mapred/TableMapReduceUtil.java | 2 +- .../apache/hadoop/hbase/mapreduce/Import.java | 6 ++-- .../hadoop/hbase/mapreduce/ImportTsv.java | 2 +- .../hbase/mapreduce/TableMapReduceUtil.java | 30 +++++++++++++++++-- .../hadoop/hbase/mapreduce/WALPlayer.java | 4 +-- .../hadoop/hbase/PerformanceEvaluation.java | 2 +- .../mapred/TestTableSnapshotInputFormat.java | 2 +- .../TestTableSnapshotInputFormat.java | 2 +- src/main/asciidoc/_chapters/mapreduce.adoc | 2 +- 12 files changed, 45 insertions(+), 19 deletions(-) diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java index 6c54dcaeac4..135bea75f50 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java @@ -785,7 +785,8 @@ public class IntegrationTestBigLinkedList extends IntegrationTestBase { job.getConfiguration().setBoolean("mapreduce.map.speculative", false); TableMapReduceUtil.addDependencyJars(job); - TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class); + TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), + AbstractHBaseTool.class); TableMapReduceUtil.initCredentials(job); boolean success = jobCompletion(job); @@ -1296,7 +1297,8 @@ public class IntegrationTestBigLinkedList extends IntegrationTestBase { TableMapReduceUtil.initTableMapperJob(getTableName(getConf()).getName(), scan, VerifyMapper.class, BytesWritable.class, BytesWritable.class, job); - TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class); + TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), + AbstractHBaseTool.class); job.getConfiguration().setBoolean("mapreduce.map.speculative", false); diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java index e279dfb6e27..d12383d2ec4 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.java @@ -348,7 +348,7 @@ public void cleanUpCluster() throws Exception { TableMapReduceUtil.addDependencyJars(job); - TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class); + TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), AbstractHBaseTool.class); TableMapReduceUtil.initCredentials(job); assertTrue(job.waitForCompletion(true)); return job; @@ -372,7 +372,7 @@ public void cleanUpCluster() throws Exception { TableMapReduceUtil.initTableMapperJob( htd.getTableName().getNameAsString(), scan, VerifyMapper.class, BytesWritable.class, BytesWritable.class, job); - TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class); + TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), AbstractHBaseTool.class); int scannerCaching = conf.getInt("verify.scannercaching", SCANNER_CACHING); TableMapReduceUtil.setScannerCaching(job, scannerCaching); diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestWithCellVisibilityLoadAndVerify.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestWithCellVisibilityLoadAndVerify.java index 52a705bb3b4..abaa48142b9 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestWithCellVisibilityLoadAndVerify.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestWithCellVisibilityLoadAndVerify.java @@ -331,7 +331,7 @@ public class IntegrationTestWithCellVisibilityLoadAndVerify extends IntegrationT scan.setAuthorizations(new Authorizations(auths)); TableMapReduceUtil.initTableMapperJob(htd.getTableName().getNameAsString(), scan, VerifyMapper.class, NullWritable.class, NullWritable.class, job); - TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class); + TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), AbstractHBaseTool.class); int scannerCaching = conf.getInt("verify.scannercaching", SCANNER_CACHING); TableMapReduceUtil.setScannerCaching(job, scannerCaching); job.setNumReduceTasks(0); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java index 84a279d974e..73340c431c1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java @@ -362,7 +362,7 @@ public class TableMapReduceUtil { */ public static void addDependencyJars(JobConf job) throws IOException { org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addHBaseDependencyJars(job); - org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addDependencyJars( + org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addDependencyJarsForClasses( job, // when making changes here, consider also mapreduce.TableMapReduceUtil // pull job classes diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java index c7d59129a91..e2693b9d2a9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java @@ -616,7 +616,7 @@ public class Import extends Configured implements Tool { try { Class filter = conf.getClass(FILTER_CLASS_CONF_KEY, null, Filter.class); if (filter != null) { - TableMapReduceUtil.addDependencyJars(conf, filter); + TableMapReduceUtil.addDependencyJarsForClasses(conf, filter); } } catch (Exception e) { throw new IOException(e); @@ -643,7 +643,7 @@ public class Import extends Configured implements Tool { fs.deleteOnExit(partitionsPath); job.setPartitionerClass(KeyValueWritableComparablePartitioner.class); job.setNumReduceTasks(regionLocator.getStartKeys().length); - TableMapReduceUtil.addDependencyJars(job.getConfiguration(), + TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), com.google.common.base.Preconditions.class); } } else if (hfileOutPath != null) { @@ -658,7 +658,7 @@ public class Import extends Configured implements Tool { job.setMapOutputKeyClass(ImmutableBytesWritable.class); job.setMapOutputValueClass(KeyValue.class); HFileOutputFormat2.configureIncrementalLoad(job, table.getTableDescriptor(), regionLocator); - TableMapReduceUtil.addDependencyJars(job.getConfiguration(), + TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), com.google.common.base.Preconditions.class); } } else { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java index e778d99f375..39085df8237 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java @@ -587,7 +587,7 @@ public class ImportTsv extends Configured implements Tool { KeyValueSerialization.class.getName()); } TableMapReduceUtil.addDependencyJars(job); - TableMapReduceUtil.addDependencyJars(job.getConfiguration(), + TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), com.google.common.base.Function.class /* Guava used by TsvParser */); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java index 37e4e44b35d..8d8a784595b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java @@ -339,7 +339,7 @@ public class TableMapReduceUtil { if (addDependencyJars) { addDependencyJars(job); - addDependencyJars(job.getConfiguration(), MetricRegistry.class); + addDependencyJarsForClasses(job.getConfiguration(), MetricRegistry.class); } resetCacheConfig(job.getConfiguration()); @@ -787,7 +787,7 @@ public class TableMapReduceUtil { " Continuing without it."); } - addDependencyJars(conf, + addDependencyJarsForClasses(conf, // explicitly pull a class from each module org.apache.hadoop.hbase.HConstants.class, // hbase-common org.apache.hadoop.hbase.protobuf.generated.ClientProtos.class, // hbase-protocol @@ -835,7 +835,7 @@ public class TableMapReduceUtil { public static void addDependencyJars(Job job) throws IOException { addHBaseDependencyJars(job.getConfiguration()); try { - addDependencyJars(job.getConfiguration(), + addDependencyJarsForClasses(job.getConfiguration(), // when making changes here, consider also mapred.TableMapReduceUtil // pull job classes job.getMapOutputKeyClass(), @@ -855,9 +855,33 @@ public class TableMapReduceUtil { * Add the jars containing the given classes to the job's configuration * such that JobClient will ship them to the cluster and add them to * the DistributedCache. + * @deprecated rely on {@link #addDependencyJars(Job)} instead. */ + @Deprecated public static void addDependencyJars(Configuration conf, Class... classes) throws IOException { + LOG.warn("The addDependencyJars(Configuration, Class...) method has been deprecated since it" + + " is easy to use incorrectly. Most users should rely on addDependencyJars(Job) " + + "instead. See HBASE-8386 for more details."); + addDependencyJarsForClasses(conf, classes); + } + + /** + * Add the jars containing the given classes to the job's configuration + * such that JobClient will ship them to the cluster and add them to + * the DistributedCache. + * + * N.B. that this method at most adds one jar per class given. If there is more than one + * jar available containing a class with the same name as a given class, we don't define + * which of those jars might be chosen. + * + * @param conf The Hadoop Configuration to modify + * @param classes will add just those dependencies needed to find the given classes + * @throws IOException if an underlying library call fails. + */ + @InterfaceAudience.Private + public static void addDependencyJarsForClasses(Configuration conf, + Class... classes) throws IOException { FileSystem localFs = FileSystem.getLocal(conf); Set jars = new HashSet(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java index c6fefb23dd6..452714b1d19 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java @@ -282,7 +282,7 @@ public class WALPlayer extends Configured implements Tool { RegionLocator regionLocator = conn.getRegionLocator(tableName)) { HFileOutputFormat2.configureIncrementalLoad(job, table.getTableDescriptor(), regionLocator); } - TableMapReduceUtil.addDependencyJars(job.getConfiguration(), + TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), com.google.common.base.Preconditions.class); } else { // output to live cluster @@ -295,7 +295,7 @@ public class WALPlayer extends Configured implements Tool { } String codecCls = WALCellCodec.getWALCellCodecClass(conf); try { - TableMapReduceUtil.addDependencyJars(job.getConfiguration(), Class.forName(codecCls)); + TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), Class.forName(codecCls)); } catch (Exception e) { throw new IOException("Cannot determine wal codec class " + codecCls, e); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java index 0528975ef76..dc8a783461e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java @@ -499,7 +499,7 @@ public class PerformanceEvaluation extends Configured implements Tool { TextOutputFormat.setOutputPath(job, new Path(inputDir.getParent(), "outputs")); TableMapReduceUtil.addDependencyJars(job); - TableMapReduceUtil.addDependencyJars(job.getConfiguration(), + TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), Histogram.class, // yammer metrics ObjectMapper.class); // jackson-mapper-asl diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableSnapshotInputFormat.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableSnapshotInputFormat.java index c85cead4310..c4fdea41b0d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableSnapshotInputFormat.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableSnapshotInputFormat.java @@ -243,7 +243,7 @@ public class TestTableSnapshotInputFormat extends TableSnapshotInputFormatTestBa JobConf jobConf = new JobConf(util.getConfiguration()); jobConf.setJarByClass(util.getClass()); - org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addDependencyJars(jobConf, + org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addDependencyJarsForClasses(jobConf, TestTableSnapshotInputFormat.class); TableMapReduceUtil.initTableSnapshotMapJob(snapshotName, COLUMNS, diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java index 7c10defac5c..5e8c8cdcb0f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java @@ -281,7 +281,7 @@ public class TestTableSnapshotInputFormat extends TableSnapshotInputFormatTestBa Scan scan = new Scan(startRow, endRow); // limit the scan job.setJarByClass(util.getClass()); - TableMapReduceUtil.addDependencyJars(job.getConfiguration(), + TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), TestTableSnapshotInputFormat.class); TableMapReduceUtil.initTableSnapshotMapperJob(snapshotName, diff --git a/src/main/asciidoc/_chapters/mapreduce.adoc b/src/main/asciidoc/_chapters/mapreduce.adoc index 75718fd1cf7..dfa843a95fc 100644 --- a/src/main/asciidoc/_chapters/mapreduce.adoc +++ b/src/main/asciidoc/_chapters/mapreduce.adoc @@ -73,7 +73,7 @@ This example assumes you use a BASH-compatible shell. $ HADOOP_CLASSPATH=`${HBASE_HOME}/bin/hbase classpath` ${HADOOP_HOME}/bin/hadoop jar ${HBASE_HOME}/lib/hbase-server-VERSION.jar rowcounter usertable ---- -When the command runs, internally, the HBase JAR finds the dependencies it needs for ZooKeeper, Guava, and its other dependencies on the passed `HADOOP_CLASSPATH` and adds the JARs to the MapReduce job configuration. +When the command runs, internally, the HBase JAR finds the dependencies it needs and adds them to the MapReduce job configuration. See the source at `TableMapReduceUtil#addDependencyJars(org.apache.hadoop.mapreduce.Job)` for how this is done. The command `hbase mapredcp` can also help you dump the CLASSPATH entries required by MapReduce, which are the same jars `TableMapReduceUtil#addDependencyJars` would add.