HBASE-8386 deprecate TableMapReduce.addDependencyJars(Configuration, class<?> ...)
Signed-off-by: Matteo Bertozzi <matteo.bertozzi@cloudera.com>
This commit is contained in:
parent
c59f76485e
commit
431c8c9ad0
|
@ -785,7 +785,8 @@ public class IntegrationTestBigLinkedList extends IntegrationTestBase {
|
||||||
|
|
||||||
job.getConfiguration().setBoolean("mapreduce.map.speculative", false);
|
job.getConfiguration().setBoolean("mapreduce.map.speculative", false);
|
||||||
TableMapReduceUtil.addDependencyJars(job);
|
TableMapReduceUtil.addDependencyJars(job);
|
||||||
TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class);
|
TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
|
||||||
|
AbstractHBaseTool.class);
|
||||||
TableMapReduceUtil.initCredentials(job);
|
TableMapReduceUtil.initCredentials(job);
|
||||||
|
|
||||||
boolean success = jobCompletion(job);
|
boolean success = jobCompletion(job);
|
||||||
|
@ -1296,7 +1297,8 @@ public class IntegrationTestBigLinkedList extends IntegrationTestBase {
|
||||||
|
|
||||||
TableMapReduceUtil.initTableMapperJob(getTableName(getConf()).getName(), scan,
|
TableMapReduceUtil.initTableMapperJob(getTableName(getConf()).getName(), scan,
|
||||||
VerifyMapper.class, BytesWritable.class, BytesWritable.class, job);
|
VerifyMapper.class, BytesWritable.class, BytesWritable.class, job);
|
||||||
TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class);
|
TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
|
||||||
|
AbstractHBaseTool.class);
|
||||||
|
|
||||||
job.getConfiguration().setBoolean("mapreduce.map.speculative", false);
|
job.getConfiguration().setBoolean("mapreduce.map.speculative", false);
|
||||||
|
|
||||||
|
|
|
@ -348,7 +348,7 @@ public void cleanUpCluster() throws Exception {
|
||||||
|
|
||||||
TableMapReduceUtil.addDependencyJars(job);
|
TableMapReduceUtil.addDependencyJars(job);
|
||||||
|
|
||||||
TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class);
|
TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), AbstractHBaseTool.class);
|
||||||
TableMapReduceUtil.initCredentials(job);
|
TableMapReduceUtil.initCredentials(job);
|
||||||
assertTrue(job.waitForCompletion(true));
|
assertTrue(job.waitForCompletion(true));
|
||||||
return job;
|
return job;
|
||||||
|
@ -372,7 +372,7 @@ public void cleanUpCluster() throws Exception {
|
||||||
TableMapReduceUtil.initTableMapperJob(
|
TableMapReduceUtil.initTableMapperJob(
|
||||||
htd.getTableName().getNameAsString(), scan, VerifyMapper.class,
|
htd.getTableName().getNameAsString(), scan, VerifyMapper.class,
|
||||||
BytesWritable.class, BytesWritable.class, job);
|
BytesWritable.class, BytesWritable.class, job);
|
||||||
TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class);
|
TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), AbstractHBaseTool.class);
|
||||||
int scannerCaching = conf.getInt("verify.scannercaching", SCANNER_CACHING);
|
int scannerCaching = conf.getInt("verify.scannercaching", SCANNER_CACHING);
|
||||||
TableMapReduceUtil.setScannerCaching(job, scannerCaching);
|
TableMapReduceUtil.setScannerCaching(job, scannerCaching);
|
||||||
|
|
||||||
|
|
|
@ -331,7 +331,7 @@ public class IntegrationTestWithCellVisibilityLoadAndVerify extends IntegrationT
|
||||||
scan.setAuthorizations(new Authorizations(auths));
|
scan.setAuthorizations(new Authorizations(auths));
|
||||||
TableMapReduceUtil.initTableMapperJob(htd.getTableName().getNameAsString(), scan,
|
TableMapReduceUtil.initTableMapperJob(htd.getTableName().getNameAsString(), scan,
|
||||||
VerifyMapper.class, NullWritable.class, NullWritable.class, job);
|
VerifyMapper.class, NullWritable.class, NullWritable.class, job);
|
||||||
TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class);
|
TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), AbstractHBaseTool.class);
|
||||||
int scannerCaching = conf.getInt("verify.scannercaching", SCANNER_CACHING);
|
int scannerCaching = conf.getInt("verify.scannercaching", SCANNER_CACHING);
|
||||||
TableMapReduceUtil.setScannerCaching(job, scannerCaching);
|
TableMapReduceUtil.setScannerCaching(job, scannerCaching);
|
||||||
job.setNumReduceTasks(0);
|
job.setNumReduceTasks(0);
|
||||||
|
|
|
@ -362,7 +362,7 @@ public class TableMapReduceUtil {
|
||||||
*/
|
*/
|
||||||
public static void addDependencyJars(JobConf job) throws IOException {
|
public static void addDependencyJars(JobConf job) throws IOException {
|
||||||
org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addHBaseDependencyJars(job);
|
org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addHBaseDependencyJars(job);
|
||||||
org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addDependencyJars(
|
org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addDependencyJarsForClasses(
|
||||||
job,
|
job,
|
||||||
// when making changes here, consider also mapreduce.TableMapReduceUtil
|
// when making changes here, consider also mapreduce.TableMapReduceUtil
|
||||||
// pull job classes
|
// pull job classes
|
||||||
|
|
|
@ -616,7 +616,7 @@ public class Import extends Configured implements Tool {
|
||||||
try {
|
try {
|
||||||
Class<? extends Filter> filter = conf.getClass(FILTER_CLASS_CONF_KEY, null, Filter.class);
|
Class<? extends Filter> filter = conf.getClass(FILTER_CLASS_CONF_KEY, null, Filter.class);
|
||||||
if (filter != null) {
|
if (filter != null) {
|
||||||
TableMapReduceUtil.addDependencyJars(conf, filter);
|
TableMapReduceUtil.addDependencyJarsForClasses(conf, filter);
|
||||||
}
|
}
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
throw new IOException(e);
|
throw new IOException(e);
|
||||||
|
@ -643,7 +643,7 @@ public class Import extends Configured implements Tool {
|
||||||
fs.deleteOnExit(partitionsPath);
|
fs.deleteOnExit(partitionsPath);
|
||||||
job.setPartitionerClass(KeyValueWritableComparablePartitioner.class);
|
job.setPartitionerClass(KeyValueWritableComparablePartitioner.class);
|
||||||
job.setNumReduceTasks(regionLocator.getStartKeys().length);
|
job.setNumReduceTasks(regionLocator.getStartKeys().length);
|
||||||
TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
|
TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
|
||||||
com.google.common.base.Preconditions.class);
|
com.google.common.base.Preconditions.class);
|
||||||
}
|
}
|
||||||
} else if (hfileOutPath != null) {
|
} else if (hfileOutPath != null) {
|
||||||
|
@ -658,7 +658,7 @@ public class Import extends Configured implements Tool {
|
||||||
job.setMapOutputKeyClass(ImmutableBytesWritable.class);
|
job.setMapOutputKeyClass(ImmutableBytesWritable.class);
|
||||||
job.setMapOutputValueClass(KeyValue.class);
|
job.setMapOutputValueClass(KeyValue.class);
|
||||||
HFileOutputFormat2.configureIncrementalLoad(job, table.getTableDescriptor(), regionLocator);
|
HFileOutputFormat2.configureIncrementalLoad(job, table.getTableDescriptor(), regionLocator);
|
||||||
TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
|
TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
|
||||||
com.google.common.base.Preconditions.class);
|
com.google.common.base.Preconditions.class);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
|
|
@ -587,7 +587,7 @@ public class ImportTsv extends Configured implements Tool {
|
||||||
KeyValueSerialization.class.getName());
|
KeyValueSerialization.class.getName());
|
||||||
}
|
}
|
||||||
TableMapReduceUtil.addDependencyJars(job);
|
TableMapReduceUtil.addDependencyJars(job);
|
||||||
TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
|
TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
|
||||||
com.google.common.base.Function.class /* Guava used by TsvParser */);
|
com.google.common.base.Function.class /* Guava used by TsvParser */);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -339,7 +339,7 @@ public class TableMapReduceUtil {
|
||||||
|
|
||||||
if (addDependencyJars) {
|
if (addDependencyJars) {
|
||||||
addDependencyJars(job);
|
addDependencyJars(job);
|
||||||
addDependencyJars(job.getConfiguration(), MetricRegistry.class);
|
addDependencyJarsForClasses(job.getConfiguration(), MetricRegistry.class);
|
||||||
}
|
}
|
||||||
|
|
||||||
resetCacheConfig(job.getConfiguration());
|
resetCacheConfig(job.getConfiguration());
|
||||||
|
@ -787,7 +787,7 @@ public class TableMapReduceUtil {
|
||||||
" Continuing without it.");
|
" Continuing without it.");
|
||||||
}
|
}
|
||||||
|
|
||||||
addDependencyJars(conf,
|
addDependencyJarsForClasses(conf,
|
||||||
// explicitly pull a class from each module
|
// explicitly pull a class from each module
|
||||||
org.apache.hadoop.hbase.HConstants.class, // hbase-common
|
org.apache.hadoop.hbase.HConstants.class, // hbase-common
|
||||||
org.apache.hadoop.hbase.protobuf.generated.ClientProtos.class, // hbase-protocol
|
org.apache.hadoop.hbase.protobuf.generated.ClientProtos.class, // hbase-protocol
|
||||||
|
@ -835,7 +835,7 @@ public class TableMapReduceUtil {
|
||||||
public static void addDependencyJars(Job job) throws IOException {
|
public static void addDependencyJars(Job job) throws IOException {
|
||||||
addHBaseDependencyJars(job.getConfiguration());
|
addHBaseDependencyJars(job.getConfiguration());
|
||||||
try {
|
try {
|
||||||
addDependencyJars(job.getConfiguration(),
|
addDependencyJarsForClasses(job.getConfiguration(),
|
||||||
// when making changes here, consider also mapred.TableMapReduceUtil
|
// when making changes here, consider also mapred.TableMapReduceUtil
|
||||||
// pull job classes
|
// pull job classes
|
||||||
job.getMapOutputKeyClass(),
|
job.getMapOutputKeyClass(),
|
||||||
|
@ -855,9 +855,33 @@ public class TableMapReduceUtil {
|
||||||
* Add the jars containing the given classes to the job's configuration
|
* Add the jars containing the given classes to the job's configuration
|
||||||
* such that JobClient will ship them to the cluster and add them to
|
* such that JobClient will ship them to the cluster and add them to
|
||||||
* the DistributedCache.
|
* the DistributedCache.
|
||||||
|
* @deprecated rely on {@link #addDependencyJars(Job)} instead.
|
||||||
*/
|
*/
|
||||||
|
@Deprecated
|
||||||
public static void addDependencyJars(Configuration conf,
|
public static void addDependencyJars(Configuration conf,
|
||||||
Class<?>... classes) throws IOException {
|
Class<?>... classes) throws IOException {
|
||||||
|
LOG.warn("The addDependencyJars(Configuration, Class<?>...) method has been deprecated since it"
|
||||||
|
+ " is easy to use incorrectly. Most users should rely on addDependencyJars(Job) " +
|
||||||
|
"instead. See HBASE-8386 for more details.");
|
||||||
|
addDependencyJarsForClasses(conf, classes);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Add the jars containing the given classes to the job's configuration
|
||||||
|
* such that JobClient will ship them to the cluster and add them to
|
||||||
|
* the DistributedCache.
|
||||||
|
*
|
||||||
|
* N.B. that this method at most adds one jar per class given. If there is more than one
|
||||||
|
* jar available containing a class with the same name as a given class, we don't define
|
||||||
|
* which of those jars might be chosen.
|
||||||
|
*
|
||||||
|
* @param conf The Hadoop Configuration to modify
|
||||||
|
* @param classes will add just those dependencies needed to find the given classes
|
||||||
|
* @throws IOException if an underlying library call fails.
|
||||||
|
*/
|
||||||
|
@InterfaceAudience.Private
|
||||||
|
public static void addDependencyJarsForClasses(Configuration conf,
|
||||||
|
Class<?>... classes) throws IOException {
|
||||||
|
|
||||||
FileSystem localFs = FileSystem.getLocal(conf);
|
FileSystem localFs = FileSystem.getLocal(conf);
|
||||||
Set<String> jars = new HashSet<String>();
|
Set<String> jars = new HashSet<String>();
|
||||||
|
|
|
@ -282,7 +282,7 @@ public class WALPlayer extends Configured implements Tool {
|
||||||
RegionLocator regionLocator = conn.getRegionLocator(tableName)) {
|
RegionLocator regionLocator = conn.getRegionLocator(tableName)) {
|
||||||
HFileOutputFormat2.configureIncrementalLoad(job, table.getTableDescriptor(), regionLocator);
|
HFileOutputFormat2.configureIncrementalLoad(job, table.getTableDescriptor(), regionLocator);
|
||||||
}
|
}
|
||||||
TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
|
TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
|
||||||
com.google.common.base.Preconditions.class);
|
com.google.common.base.Preconditions.class);
|
||||||
} else {
|
} else {
|
||||||
// output to live cluster
|
// output to live cluster
|
||||||
|
@ -295,7 +295,7 @@ public class WALPlayer extends Configured implements Tool {
|
||||||
}
|
}
|
||||||
String codecCls = WALCellCodec.getWALCellCodecClass(conf);
|
String codecCls = WALCellCodec.getWALCellCodecClass(conf);
|
||||||
try {
|
try {
|
||||||
TableMapReduceUtil.addDependencyJars(job.getConfiguration(), Class.forName(codecCls));
|
TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), Class.forName(codecCls));
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
throw new IOException("Cannot determine wal codec class " + codecCls, e);
|
throw new IOException("Cannot determine wal codec class " + codecCls, e);
|
||||||
}
|
}
|
||||||
|
|
|
@ -499,7 +499,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
|
||||||
TextOutputFormat.setOutputPath(job, new Path(inputDir.getParent(), "outputs"));
|
TextOutputFormat.setOutputPath(job, new Path(inputDir.getParent(), "outputs"));
|
||||||
|
|
||||||
TableMapReduceUtil.addDependencyJars(job);
|
TableMapReduceUtil.addDependencyJars(job);
|
||||||
TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
|
TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
|
||||||
Histogram.class, // yammer metrics
|
Histogram.class, // yammer metrics
|
||||||
ObjectMapper.class); // jackson-mapper-asl
|
ObjectMapper.class); // jackson-mapper-asl
|
||||||
|
|
||||||
|
|
|
@ -243,7 +243,7 @@ public class TestTableSnapshotInputFormat extends TableSnapshotInputFormatTestBa
|
||||||
JobConf jobConf = new JobConf(util.getConfiguration());
|
JobConf jobConf = new JobConf(util.getConfiguration());
|
||||||
|
|
||||||
jobConf.setJarByClass(util.getClass());
|
jobConf.setJarByClass(util.getClass());
|
||||||
org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addDependencyJars(jobConf,
|
org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addDependencyJarsForClasses(jobConf,
|
||||||
TestTableSnapshotInputFormat.class);
|
TestTableSnapshotInputFormat.class);
|
||||||
|
|
||||||
TableMapReduceUtil.initTableSnapshotMapJob(snapshotName, COLUMNS,
|
TableMapReduceUtil.initTableSnapshotMapJob(snapshotName, COLUMNS,
|
||||||
|
|
|
@ -281,7 +281,7 @@ public class TestTableSnapshotInputFormat extends TableSnapshotInputFormatTestBa
|
||||||
Scan scan = new Scan(startRow, endRow); // limit the scan
|
Scan scan = new Scan(startRow, endRow); // limit the scan
|
||||||
|
|
||||||
job.setJarByClass(util.getClass());
|
job.setJarByClass(util.getClass());
|
||||||
TableMapReduceUtil.addDependencyJars(job.getConfiguration(),
|
TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
|
||||||
TestTableSnapshotInputFormat.class);
|
TestTableSnapshotInputFormat.class);
|
||||||
|
|
||||||
TableMapReduceUtil.initTableSnapshotMapperJob(snapshotName,
|
TableMapReduceUtil.initTableSnapshotMapperJob(snapshotName,
|
||||||
|
|
|
@ -73,7 +73,7 @@ This example assumes you use a BASH-compatible shell.
|
||||||
$ HADOOP_CLASSPATH=`${HBASE_HOME}/bin/hbase classpath` ${HADOOP_HOME}/bin/hadoop jar ${HBASE_HOME}/lib/hbase-server-VERSION.jar rowcounter usertable
|
$ HADOOP_CLASSPATH=`${HBASE_HOME}/bin/hbase classpath` ${HADOOP_HOME}/bin/hadoop jar ${HBASE_HOME}/lib/hbase-server-VERSION.jar rowcounter usertable
|
||||||
----
|
----
|
||||||
|
|
||||||
When the command runs, internally, the HBase JAR finds the dependencies it needs for ZooKeeper, Guava, and its other dependencies on the passed `HADOOP_CLASSPATH` and adds the JARs to the MapReduce job configuration.
|
When the command runs, internally, the HBase JAR finds the dependencies it needs and adds them to the MapReduce job configuration.
|
||||||
See the source at `TableMapReduceUtil#addDependencyJars(org.apache.hadoop.mapreduce.Job)` for how this is done.
|
See the source at `TableMapReduceUtil#addDependencyJars(org.apache.hadoop.mapreduce.Job)` for how this is done.
|
||||||
|
|
||||||
The command `hbase mapredcp` can also help you dump the CLASSPATH entries required by MapReduce, which are the same jars `TableMapReduceUtil#addDependencyJars` would add.
|
The command `hbase mapredcp` can also help you dump the CLASSPATH entries required by MapReduce, which are the same jars `TableMapReduceUtil#addDependencyJars` would add.
|
||||||
|
|
Loading…
Reference in New Issue