Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-12798 Map Reduce jobs should not create Tables in setConf() (Solomon Duskis)

This commit is contained in:
tedyu 2015-01-10 14:23:03 -08:00
parent 988cba762a
commit f6a017ce63
3 changed files with 187 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormat.java
View File

@ -49,6 +49,7 @@ import org.apache.hadoop.util.StringUtils;
public class TableInputFormat extends TableInputFormatBase
implements Configurable {
@SuppressWarnings("hiding")
private static final Log LOG = LogFactory.getLog(TableInputFormat.class);
/** Job parameter that specifies the input table. */
@ -112,13 +113,6 @@ implements Configurable {
@Override
public void setConf(Configuration configuration) {
this.conf = configuration;
TableName tableName = TableName.valueOf(conf.get(INPUT_TABLE));
try {
// NOTE: This connection doesn't currently get closed explicit1ly.
initializeTable(ConnectionFactory.createConnection(new Configuration(conf)), tableName);
} catch (Exception e) {
LOG.error(StringUtils.stringifyException(e));
}
Scan scan = null;
@ -180,6 +174,16 @@ implements Configurable {
setScan(scan);
}
@Override
protected void initialize() {
TableName tableName = TableName.valueOf(conf.get(INPUT_TABLE));
try {
initializeTable(ConnectionFactory.createConnection(new Configuration(conf)), tableName);
} catch (Exception e) {
LOG.error(StringUtils.stringifyException(e));
}
}
/**
* Parses a combined family and qualifier and adds either both or just the
* family in case there is no qualifier. This assumes the older colon

116
hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
View File

@ -67,12 +67,10 @@ import org.apache.hadoop.util.StringUtils;
* <pre>
* class ExampleTIF extends TableInputFormatBase implements JobConfigurable {
*
* private JobConf job;
*
* public void configure(JobConf job) {
* Connection connection =
* ConnectionFactory.createConnection(HBaseConfiguration.create(job));
* TableName tableName = TableName.valueOf("exampleTable");
* // mandatory
* initializeTable(connection, tableName);
* this.job = job;
* Text[] inputColumns = new byte [][] { Bytes.toBytes("cf1:columnA"),
* Bytes.toBytes("cf2") };
* // mandatory
@ -82,6 +80,14 @@ import org.apache.hadoop.util.StringUtils;
* setRowFilter(exampleFilter);
* }
*
* protected void initialize() {
* Connection connection =
* ConnectionFactory.createConnection(HBaseConfiguration.create(job));
* TableName tableName = TableName.valueOf("exampleTable");
* // mandatory
* initializeTable(connection, tableName);
* }
*
* public void validateInput(JobConf job) throws IOException {
* }
* }
@ -116,13 +122,14 @@ extends InputFormat<ImmutableBytesWritable, Result> {
private RegionLocator regionLocator;
/** The reader scanning the table, can be a custom one. */
private TableRecordReader tableRecordReader = null;
/** The underlying {@link Connection} of the table. */
private Connection connection;
/** The reverse DNS lookup cache mapping: IPAddress => HostName */
private HashMap<InetAddress, String> reverseDNSCacheMap =
new HashMap<InetAddress, String>();
private Connection connection;
/**
* Builds a {@link TableRecordReader}. If no {@link TableRecordReader} was provided, uses
* the default.
@ -140,6 +147,10 @@ extends InputFormat<ImmutableBytesWritable, Result> {
InputSplit split, TaskAttemptContext context)
throws IOException {
if (table == null) {
initialize();
}
if (getTable() == null) {
// initialize() must not have been implemented in the subclass.
throw new IOException("Cannot create a record reader because of a" +
" previous error. Please look at the previous logs lines from" +
" the task's full log for more details.");
@ -152,19 +163,13 @@ extends InputFormat<ImmutableBytesWritable, Result> {
sc.setStartRow(tSplit.getStartRow());
sc.setStopRow(tSplit.getEndRow());
trr.setScan(sc);
trr.setTable(table);
trr.setTable(getTable());
return new RecordReader<ImmutableBytesWritable, Result>() {
@Override
public void close() throws IOException {
trr.close();
close(admin, table, regionLocator, connection);
}
private void close(Closeable... closables) throws IOException {
for (Closeable c : closables) {
if(c != null) { c.close(); }
}
closeTable();
}
@Override
@ -196,7 +201,7 @@ extends InputFormat<ImmutableBytesWritable, Result> {
}
protected Pair<byte[][],byte[][]> getStartEndKeys() throws IOException {
return regionLocator.getStartEndKeys();
return getRegionLocator().getStartEndKeys();
}
/**
@ -211,22 +216,35 @@ extends InputFormat<ImmutableBytesWritable, Result> {
*/
@Override
public List<InputSplit> getSplits(JobContext context) throws IOException {
boolean closeOnFinish = false;
if (table == null) {
initialize();
closeOnFinish = true;
}
if (getTable() == null) {
// initialize() wasn't implemented, so the table is null.
throw new IOException("No table was provided.");
}
RegionSizeCalculator sizeCalculator = new RegionSizeCalculator(regionLocator, admin);
try {
RegionSizeCalculator sizeCalculator =
new RegionSizeCalculator(getRegionLocator(), getAdmin());
TableName tableName = getTable().getName();
Pair<byte[][], byte[][]> keys = getStartEndKeys();
if (keys == null || keys.getFirst() == null ||
keys.getFirst().length == 0) {
HRegionLocation regLoc = regionLocator.getRegionLocation(HConstants.EMPTY_BYTE_ARRAY, false);
HRegionLocation regLoc =
getRegionLocator().getRegionLocation(HConstants.EMPTY_BYTE_ARRAY, false);
if (null == regLoc) {
throw new IOException("Expecting at least one region.");
}
List<InputSplit> splits = new ArrayList<InputSplit>(1);
long regionSize = sizeCalculator.getRegionSize(regLoc.getRegionInfo().getRegionName());
TableSplit split = new TableSplit(table.getName(),
TableSplit split = new TableSplit(tableName,
HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY, regLoc
.getHostnamePort().split(Addressing.HOSTNAME_PORT_SEPARATOR)[0], regionSize);
splits.add(split);
@ -237,7 +255,7 @@ extends InputFormat<ImmutableBytesWritable, Result> {
if (!includeRegionInSplit(keys.getFirst()[i], keys.getSecond()[i])) {
continue;
}
HRegionLocation location = regionLocator.getRegionLocation(keys.getFirst()[i], false);
HRegionLocation location = getRegionLocator().getRegionLocation(keys.getFirst()[i], false);
// The below InetSocketAddress creation does a name resolution.
InetSocketAddress isa = new InetSocketAddress(location.getHostname(), location.getPort());
if (isa.isUnresolved()) {
@ -269,7 +287,7 @@ extends InputFormat<ImmutableBytesWritable, Result> {
byte[] regionName = location.getRegionInfo().getRegionName();
long regionSize = sizeCalculator.getRegionSize(regionName);
TableSplit split = new TableSplit(table.getName(),
TableSplit split = new TableSplit(tableName,
splitStart, splitStop, regionLocation, regionSize);
splits.add(split);
if (LOG.isDebugEnabled()) {
@ -297,6 +315,11 @@ extends InputFormat<ImmutableBytesWritable, Result> {
} else {
return splits;
}
} finally {
if (closeOnFinish) {
closeTable();
}
}
}
public String reverseDNS(InetAddress ipAddress) throws NamingException, UnknownHostException {
@ -343,6 +366,7 @@ extends InputFormat<ImmutableBytesWritable, Result> {
int count = 0;
while (count < list.size()) {
TableSplit ts = (TableSplit)list.get(count);
TableName tableName = ts.getTable();
String regionLocation = ts.getRegionLocation();
long regionSize = ts.getLength();
if (regionSize >= dataSkewThreshold) {
@ -351,9 +375,9 @@ extends InputFormat<ImmutableBytesWritable, Result> {
byte[] splitKey = getSplitKey(ts.getStartRow(), ts.getEndRow(), isTextKey);
//Set the size of child TableSplit as 1/2 of the region size. The exact size of the
// MapReduce input splits is not far off.
TableSplit t1 = new TableSplit(table.getName(), ts.getStartRow(), splitKey, regionLocation,
TableSplit t1 = new TableSplit(tableName, ts.getStartRow(), splitKey, regionLocation,
regionSize / 2);
TableSplit t2 = new TableSplit(table.getName(), splitKey, ts.getEndRow(), regionLocation,
TableSplit t2 = new TableSplit(tableName, splitKey, ts.getEndRow(), regionLocation,
regionSize - regionSize / 2);
resultList.add(t1);
resultList.add(t2);
@ -380,7 +404,7 @@ extends InputFormat<ImmutableBytesWritable, Result> {
break;
}
}
TableSplit t = new TableSplit(table.getName(), splitStartKey, splitEndKey,
TableSplit t = new TableSplit(tableName, splitStartKey, splitEndKey,
regionLocation, totalSize);
resultList.add(t);
}
@ -515,13 +539,16 @@ extends InputFormat<ImmutableBytesWritable, Result> {
*/
@Deprecated
protected HTable getHTable() {
return (HTable) this.table;
return (HTable) this.getTable();
}
/**
* Allows subclasses to get the {@link RegionLocator}.
*/
protected RegionLocator getRegionLocator() {
if (regionLocator == null) {
initialize();
}
return regionLocator;
}
@ -529,6 +556,9 @@ extends InputFormat<ImmutableBytesWritable, Result> {
* Allows subclasses to get the {@link Table}.
*/
protected Table getTable() {
if (table == null) {
initialize();
}
return table;
}
@ -536,6 +566,9 @@ extends InputFormat<ImmutableBytesWritable, Result> {
* Allows subclasses to get the {@link Admin}.
*/
protected Admin getAdmin() {
if (admin == null) {
initialize();
}
return admin;
}
@ -550,7 +583,8 @@ extends InputFormat<ImmutableBytesWritable, Result> {
protected void setHTable(HTable table) throws IOException {
this.table = table;
this.regionLocator = table.getRegionLocator();
this.admin = table.getConnection().getAdmin();
this.connection = table.getConnection();
this.admin = this.connection.getAdmin();
}
/**
@ -595,4 +629,34 @@ extends InputFormat<ImmutableBytesWritable, Result> {
protected void setTableRecordReader(TableRecordReader tableRecordReader) {
this.tableRecordReader = tableRecordReader;
}
/**
* This method will be called when any of the following are referenced, but not yet initialized:
* admin, regionLocator, table. Subclasses will have the opportunity to call
* {@link #initializeTable(Connection, TableName)}
*/
protected void initialize() {
}
/**
* Close the Table and related objects that were initialized via
* {@link #initializeTable(Connection, TableName)}.
*
* @throws IOException
*/
protected void closeTable() throws IOException {
close(admin, table, regionLocator, connection);
admin = null;
table = null;
regionLocator = null;
connection = null;
}
private void close(Closeable... closables) throws IOException {
for (Closeable c : closables) {
if(c != null) { c.close(); }
}
}
}

22
hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java
View File

@ -78,15 +78,26 @@ implements Configurable {
/** The configuration. */
private Configuration conf = null;
private Table table;
private Connection connection;
/**
* Writes the reducer output to an HBase table.
*/
protected class TableRecordWriter
extends RecordWriter<KEY, Mutation> {
private Connection connection;
private Table table;
/**
* @throws IOException
*
*/
public TableRecordWriter() throws IOException {
String tableName = conf.get(OUTPUT_TABLE);
this.connection = ConnectionFactory.createConnection(conf);
this.table = connection.getTable(TableName.valueOf(tableName));
this.table.setAutoFlushTo(false);
LOG.info("Created table instance for " + tableName);
}
/**
* Closes the writer, in this case flush table commits.
*
@ -164,6 +175,7 @@ implements Configurable {
return new TableOutputCommitter();
}
@Override
public Configuration getConf() {
return conf;
}
@ -192,10 +204,6 @@ implements Configurable {
if (zkClientPort != 0) {
this.conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zkClientPort);
}
this.connection = ConnectionFactory.createConnection(this.conf);
this.table = connection.getTable(TableName.valueOf(tableName));
this.table.setAutoFlushTo(false);
LOG.info("Created table instance for " + tableName);
} catch(IOException e) {
LOG.error(e);
throw new RuntimeException(e);