diff --git a/CHANGES.txt b/CHANGES.txt index c2788f3c98f..d6790aa559f 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -926,6 +926,7 @@ Release 0.21.0 - Unreleased HBASE-2782 QOS for META table access HBASE-3017 More log pruning HBASE-3022 Change format of enum messages in o.a.h.h.executor package + HBASE-3001 Ship dependency jars to the cluster for all jobs NEW FEATURES HBASE-1961 HBase EC2 scripts diff --git a/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java b/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java index 41748fe6de3..6e7526b2578 100644 --- a/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java +++ b/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java @@ -29,6 +29,10 @@ import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapred.FileInputFormat; import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.InputFormat; +import org.apache.hadoop.mapred.OutputFormat; +import org.apache.hadoop.mapred.TextInputFormat; +import org.apache.hadoop.mapred.TextOutputFormat; /** * Utility for {@link TableMap} and {@link TableReduce} @@ -59,6 +63,11 @@ public class TableMapReduceUtil { job.setMapperClass(mapper); FileInputFormat.addInputPaths(job, table); job.set(TableInputFormat.COLUMN_LIST, columns); + try { + addDependencyJars(job); + } catch (IOException e) { + e.printStackTrace(); + } } /** @@ -105,6 +114,7 @@ public class TableMapReduceUtil { } else if (partitioner != null) { job.setPartitionerClass(partitioner); } + addDependencyJars(job); } /** @@ -181,4 +191,22 @@ public class TableMapReduceUtil { public static void setScannerCaching(JobConf job, int batchSize) { job.setInt("hbase.client.scanner.caching", batchSize); } -} \ No newline at end of file + + /** + * @see org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil#addDependencyJars(Job) + */ + public static void addDependencyJars(JobConf job) throws IOException { + org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.addDependencyJars( + job, + org.apache.zookeeper.ZooKeeper.class, + com.google.common.base.Function.class, + job.getMapOutputKeyClass(), + job.getMapOutputValueClass(), + job.getOutputKeyClass(), + job.getOutputValueClass(), + job.getPartitionerClass(), + job.getClass("mapred.input.format.class", TextInputFormat.class, InputFormat.class), + job.getClass("mapred.output.format.class", TextOutputFormat.class, OutputFormat.class), + job.getCombinerClass()); + } +} diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java index ff0c542e1d3..22533e14334 100644 --- a/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java +++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java @@ -47,9 +47,6 @@ import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.zookeeper.ZooKeeper; - -import com.google.common.base.Function; /** * Utility for {@link TableMapper} and {@link TableReducer} @@ -81,6 +78,7 @@ public class TableMapReduceUtil { job.getConfiguration().set(TableInputFormat.INPUT_TABLE, table); job.getConfiguration().set(TableInputFormat.SCAN, convertScanToString(scan)); + addDependencyJars(job); } /** @@ -192,6 +190,7 @@ public class TableMapReduceUtil { } else if (partitioner != null) { job.setPartitionerClass(partitioner); } + addDependencyJars(job); } /** @@ -246,10 +245,11 @@ public class TableMapReduceUtil { public static void addDependencyJars(Job job) throws IOException { try { addDependencyJars(job.getConfiguration(), - ZooKeeper.class, - Function.class, // Guava collections + org.apache.zookeeper.ZooKeeper.class, + com.google.common.base.Function.class, job.getMapOutputKeyClass(), job.getMapOutputValueClass(), + job.getInputFormatClass(), job.getOutputKeyClass(), job.getOutputValueClass(), job.getOutputFormatClass(), @@ -267,38 +267,38 @@ public class TableMapReduceUtil { */ public static void addDependencyJars(Configuration conf, Class... classes) throws IOException { - + FileSystem localFs = FileSystem.getLocal(conf); Set jars = new HashSet(); + + // Add jars that are already in the tmpjars variable + jars.addAll( conf.getStringCollection("tmpjars") ); + + // Add jars containing the specified classes for (Class clazz : classes) { if (clazz == null) continue; - + String pathStr = findContainingJar(clazz); if (pathStr == null) { LOG.warn("Could not find jar for class " + clazz + - " in order to ship it to the cluster."); + " in order to ship it to the cluster."); continue; } Path path = new Path(pathStr); if (!localFs.exists(path)) { LOG.warn("Could not validate jar file " + path + " for class " - + clazz); + + clazz); continue; } jars.add(path.makeQualified(localFs).toString()); } if (jars.isEmpty()) return; - - String tmpJars = conf.get("tmpjars"); - if (tmpJars == null) { - tmpJars = StringUtils.arrayToString(jars.toArray(new String[0])); - } else { - tmpJars += "," + StringUtils.arrayToString(jars.toArray(new String[0])); - } - conf.set("tmpjars", tmpJars); + + conf.set("tmpjars", + StringUtils.arrayToString(jars.toArray(new String[0]))); } - + /** * Find a jar that contains a class of the same name, if any. * It will return a jar file, even if that is not the first thing diff --git a/src/main/java/org/apache/hadoop/hbase/mapreduce/package-info.java b/src/main/java/org/apache/hadoop/hbase/mapreduce/package-info.java index 62537ef0fe2..fec0dacfba9 100644 --- a/src/main/java/org/apache/hadoop/hbase/mapreduce/package-info.java +++ b/src/main/java/org/apache/hadoop/hbase/mapreduce/package-info.java @@ -1,5 +1,5 @@ /* - * Copyright 2008 The Apache Software Foundation + * Copyright 20010 The Apache Software Foundation * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -24,6 +24,7 @@ Input/OutputFormats, a table indexing MapReduce job, and utility

Table of Contents