Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-1333 RowCounter updates 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@767802 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael Stack 2009-04-23 05:29:10 +00:00
parent 9a5f753e95
commit 6f966f2caa
2 changed files with 38 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -151,6 +151,7 @@ Release 0.20.0 - Unreleased
HBASE-1331 Lower the default scanner caching value
HBASE-1235 Add table enabled status to shell and UI
(Lars George via Stack)
HBASE-1333 RowCounter updates
Release 0.19.0 - 01/21/2009
INCOMPATIBLE CHANGES

80
src/java/org/apache/hadoop/hbase/mapred/RowCounter.java
View File

@ -22,18 +22,15 @@ package org.apache.hadoop.hbase.mapred;
import java.io.IOException;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.io.HbaseMapWritable;
import org.apache.hadoop.hbase.io.Cell;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.io.RowResult;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MapReduceBase;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.lib.IdentityReducer;
@ -45,36 +42,41 @@ import org.apache.hadoop.util.ToolRunner;
* Map outputs table rows IF the input row has columns that have content.
* Uses an {@link IdentityReducer}
*/
public class RowCounter
extends MapReduceBase
implements TableMap<ImmutableBytesWritable, RowResult>, Tool {
/* Name of this 'program'
*/
public class RowCounter extends Configured implements Tool {
// Name of this 'program'
static final String NAME = "rowcounter";
private Configuration conf;
private final RowResult EMPTY_RESULT_VALUE =
new RowResult(Bytes.toBytes("dummy"),new HbaseMapWritable<byte [], Cell>());
private static enum Counters {ROWS}
public void map(ImmutableBytesWritable row, RowResult value,
OutputCollector<ImmutableBytesWritable, RowResult> output,
Reporter reporter)
throws IOException {
boolean content = false;
for (Map.Entry<byte [], Cell> e: value.entrySet()) {
Cell cell = e.getValue();
if (cell != null && cell.getValue().length > 0) {
content = true;
break;
static class RowCounterMapper
implements TableMap<ImmutableBytesWritable, RowResult> {
private static enum Counters {ROWS}
public void map(ImmutableBytesWritable row, RowResult value,
OutputCollector<ImmutableBytesWritable, RowResult> output,
Reporter reporter)
throws IOException {
boolean content = false;
for (Map.Entry<byte [], Cell> e: value.entrySet()) {
Cell cell = e.getValue();
if (cell != null && cell.getValue().length > 0) {
content = true;
break;
}
}
if (!content) {
// Don't count rows that are all empty values.
return;
}
// Give out same value every time. We're only interested in the row/key
reporter.incrCounter(Counters.ROWS, 1);
}
if (!content) {
return;
public void configure(JobConf jc) {
// Nothing to do.
}
public void close() throws IOException {
// Nothing to do.
}
// Give out same value every time. We're only interested in the row/key
reporter.incrCounter(Counters.ROWS, 1);
output.collect(row, EMPTY_RESULT_VALUE);
}
/**
@ -83,7 +85,7 @@ implements TableMap<ImmutableBytesWritable, RowResult>, Tool {
* @throws IOException
*/
public JobConf createSubmittableJob(String[] args) throws IOException {
JobConf c = new JobConf(getConf(), RowCounter.class);
JobConf c = new JobConf(getConf(), getClass());
c.setJobName(NAME);
// Columns are space delimited
StringBuilder sb = new StringBuilder();
@ -95,9 +97,9 @@ implements TableMap<ImmutableBytesWritable, RowResult>, Tool {
sb.append(args[i]);
}
// Second argument is the table name.
TableMapReduceUtil.initTableMapJob(args[1], sb.toString(), this.getClass(),
ImmutableBytesWritable.class, RowResult.class, c);
c.setReducerClass(IdentityReducer.class);
TableMapReduceUtil.initTableMapJob(args[1], sb.toString(),
RowCounterMapper.class, ImmutableBytesWritable.class, RowResult.class, c);
c.setNumReduceTasks(0);
// First arg is the output directory.
FileOutputFormat.setOutputPath(c, new Path(args[0]));
return c;
@ -119,14 +121,6 @@ implements TableMap<ImmutableBytesWritable, RowResult>, Tool {
return 0;
}
public Configuration getConf() {
return this.conf;
}
public void setConf(final Configuration c) {
this.conf = c;
}
/**
* @param args
* @throws Exception
@ -136,4 +130,4 @@ implements TableMap<ImmutableBytesWritable, RowResult>, Tool {
int errCode = ToolRunner.run(c, new RowCounter(), args);
System.exit(errCode);
}
}
}