Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-15223 Make convertScanToString public for Spark

This commit is contained in:
Jerry He 2016-02-10 15:02:58 -08:00
parent abb6cdce71
commit 1942a99b83
3 changed files with 19 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
View File

@ -266,7 +266,7 @@ extends InputFormat<ImmutableBytesWritable, Result> {
} }
List<InputSplit> splits = new ArrayList<InputSplit>(1); List<InputSplit> splits = new ArrayList<InputSplit>(1);
long regionSize = sizeCalculator.getRegionSize(regLoc.getRegionInfo().getRegionName()); long regionSize = sizeCalculator.getRegionSize(regLoc.getRegionInfo().getRegionName());
TableSplit split = new TableSplit(tableName, TableSplit split = new TableSplit(tableName, scan,
HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY, regLoc HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY, regLoc
.getHostnamePort().split(Addressing.HOSTNAME_PORT_SEPARATOR)[0], regionSize); .getHostnamePort().split(Addressing.HOSTNAME_PORT_SEPARATOR)[0], regionSize);
splits.add(split); splits.add(split);
@ -309,7 +309,7 @@ extends InputFormat<ImmutableBytesWritable, Result> {
byte[] regionName = location.getRegionInfo().getRegionName(); byte[] regionName = location.getRegionInfo().getRegionName();
long regionSize = sizeCalculator.getRegionSize(regionName); long regionSize = sizeCalculator.getRegionSize(regionName);
TableSplit split = new TableSplit(tableName, TableSplit split = new TableSplit(tableName, scan,
splitStart, splitStop, regionLocation, regionSize); splitStart, splitStop, regionLocation, regionSize);
splits.add(split); splits.add(split);
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
@ -397,9 +397,9 @@ extends InputFormat<ImmutableBytesWritable, Result> {
byte[] splitKey = getSplitKey(ts.getStartRow(), ts.getEndRow(), isTextKey); byte[] splitKey = getSplitKey(ts.getStartRow(), ts.getEndRow(), isTextKey);
//Set the size of child TableSplit as 1/2 of the region size. The exact size of the //Set the size of child TableSplit as 1/2 of the region size. The exact size of the
// MapReduce input splits is not far off. // MapReduce input splits is not far off.
TableSplit t1 = new TableSplit(tableName, ts.getStartRow(), splitKey, regionLocation, TableSplit t1 = new TableSplit(tableName, scan, ts.getStartRow(), splitKey, regionLocation,
regionSize / 2); regionSize / 2);
TableSplit t2 = new TableSplit(tableName, splitKey, ts.getEndRow(), regionLocation, TableSplit t2 = new TableSplit(tableName, scan, splitKey, ts.getEndRow(), regionLocation,
regionSize - regionSize / 2); regionSize - regionSize / 2);
resultList.add(t1); resultList.add(t1);
resultList.add(t2); resultList.add(t2);
@ -426,7 +426,7 @@ extends InputFormat<ImmutableBytesWritable, Result> {
break; break;
} }
} }
TableSplit t = new TableSplit(tableName, splitStartKey, splitEndKey, TableSplit t = new TableSplit(tableName, scan, splitStartKey, splitEndKey,
regionLocation, totalSize); regionLocation, totalSize);
resultList.add(t); resultList.add(t);
} }

4
hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
View File

@ -561,7 +561,7 @@ public class TableMapReduceUtil {
* @return The scan saved in a Base64 encoded string. * @return The scan saved in a Base64 encoded string.
* @throws IOException When writing the scan fails. * @throws IOException When writing the scan fails.
*/ */
static String convertScanToString(Scan scan) throws IOException { public static String convertScanToString(Scan scan) throws IOException {
ClientProtos.Scan proto = ProtobufUtil.toScan(scan); ClientProtos.Scan proto = ProtobufUtil.toScan(scan);
return Base64.encodeBytes(proto.toByteArray()); return Base64.encodeBytes(proto.toByteArray());
} }
@ -573,7 +573,7 @@ public class TableMapReduceUtil {
* @return The newly created Scan instance. * @return The newly created Scan instance.
* @throws IOException When reading the scan instance fails. * @throws IOException When reading the scan instance fails.
*/ */
static Scan convertStringToScan(String base64) throws IOException { public static Scan convertStringToScan(String base64) throws IOException {
byte [] decoded = Base64.decode(base64); byte [] decoded = Base64.decode(base64);
ClientProtos.Scan scan; ClientProtos.Scan scan;
try { try {

13
hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSplit.java
View File

@ -301,7 +301,18 @@ implements Writable, Comparable<TableSplit> {
StringBuilder sb = new StringBuilder(); StringBuilder sb = new StringBuilder();
sb.append("HBase table split("); sb.append("HBase table split(");
sb.append("table name: ").append(tableName); sb.append("table name: ").append(tableName);
sb.append(", scan: ").append(scan); // null scan input is represented by ""
String printScan = "";
if (!scan.equals("")) {
try {
// get the real scan here in toString, not the Base64 string
printScan = TableMapReduceUtil.convertStringToScan(scan).toString();
}
catch (IOException e) {
printScan = "";
}
}
sb.append(", scan: ").append(printScan);
sb.append(", start row: ").append(Bytes.toStringBinary(startRow)); sb.append(", start row: ").append(Bytes.toStringBinary(startRow));
sb.append(", end row: ").append(Bytes.toStringBinary(endRow)); sb.append(", end row: ").append(Bytes.toStringBinary(endRow));
sb.append(", region location: ").append(regionLocation); sb.append(", region location: ").append(regionLocation);