Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-1541, 1488 (thrift server only) 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@786259 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Ryan Rawson 2009-06-18 20:47:03 +00:00
parent 83b3772eb3
commit 4cd468ff51
12 changed files with 184 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGES.txt
View File

@ -358,6 +358,9 @@ Release 0.20.0 - Unreleased
HBASE-1538 Up zookeeper timeout from 10 seconds to 30 seconds to cut down
on hbase-user traffic
HBASE-1539 prevent aborts due to missing zoo.cfg
HBASE-1488 Fix TestThriftServer and re-enable it
HBASE-1541 Scanning multiple column families in the presence of deleted
families results in bad scans
OPTIMIZATIONS
HBASE-1412 Change values for delete column and column family in KeyValue

8
src/java/org/apache/hadoop/hbase/client/Delete.java
View File

@ -104,6 +104,14 @@ public class Delete implements Writable {
}
}
/**
* Method to check if the familyMap is empty
* @return true if empty, false otherwise
*/
public boolean isEmpty() {
return familyMap.isEmpty();
}
/**
* Delete all versions of all columns of the specified family.
* <p>

2
src/java/org/apache/hadoop/hbase/client/Result.java
View File

@ -235,6 +235,8 @@ public class Result implements Writable {
public Cell getCellValue(byte[] family, byte[] qualifier) {
Map.Entry<Long,byte[]> val = getKeyValue(family, qualifier);
if (val == null)
return null;
return new Cell(val.getValue(), val.getKey());
}

10
src/java/org/apache/hadoop/hbase/regionserver/DeleteCompare.java
View File

@ -89,8 +89,8 @@ public class DeleteCompare {
int timeRes = Bytes.compareTo(memBuffer, tsOffset, Bytes.SIZEOF_LONG,
deleteBuffer, deleteTimeOffset, Bytes.SIZEOF_LONG);
if(deleteType == KeyValue.Type.DeleteFamily.getCode()) {
if(timeRes <= 0){
if (deleteType == KeyValue.Type.DeleteFamily.getCode()) {
if (timeRes <= 0) {
return DeleteCode.DELETE;
}
return DeleteCode.SKIP;
@ -99,16 +99,16 @@ public class DeleteCompare {
//Compare columns
res = Bytes.compareTo(memBuffer, memOffset, memQualifierLen,
deleteBuffer, deleteQualifierOffset, deleteQualifierLength);
if(res < 0) {
if (res < 0) {
return DeleteCode.SKIP;
} else if(res > 0) {
return DeleteCode.DONE;
}
// same column, compare the time.
if(timeRes == 0) {
if (timeRes == 0) {
return DeleteCode.DELETE;
} else if (timeRes < 0) {
if(deleteType == KeyValue.Type.DeleteColumn.getCode()) {
if (deleteType == KeyValue.Type.DeleteColumn.getCode()) {
return DeleteCode.DELETE;
}
return DeleteCode.DONE;

17
src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
View File

@ -1710,7 +1710,7 @@ public class HRegion implements HConstants { // , Writable{
if(stopRow != null &&
comparator.compareRows(stopRow, 0, stopRow.length,
currentRow, 0, currentRow.length)
<= 0){
<= 0) {
return false;
}
this.storeHeap.next(results);
@ -1721,6 +1721,21 @@ public class HRegion implements HConstants { // , Writable{
}
byte [] row = kv.getRow();
if(!Bytes.equals(currentRow, row)) {
// Next row:
// what happens if there are _no_ results:
if (results.isEmpty()) {
// Continue on the next row:
currentRow = row;
// But did we pass the stop row?
if (stopRow != null &&
comparator.compareRows(stopRow, 0, stopRow.length,
currentRow, 0, currentRow.length) <= 0) {
return false;
}
continue;
}
return true;
}
this.storeHeap.next(results);

4
src/java/org/apache/hadoop/hbase/regionserver/Memcache.java
View File

@ -251,13 +251,13 @@ class Memcache {
byte deleteType = deleteBuffer[deleteOffset];
//Comparing with tail from memcache
for(KeyValue mem : tailSet) {
for (KeyValue mem : tailSet) {
DeleteCode res = DeleteCompare.deleteCompare(mem, deleteBuffer,
deleteRowOffset, deleteRowLen, deleteQualifierOffset,
deleteQualifierLen, deleteTimestampOffset, deleteType,
comparator.getRawComparator());
if(res == DeleteCode.DONE) {
if (res == DeleteCode.DONE) {
break;
} else if (res == DeleteCode.DELETE) {
deletes.add(mem);

30
src/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
View File

@ -123,7 +123,8 @@ class StoreScanner implements KeyValueScanner, InternalScanner, ChangedReadersOb
* @param result
* @return true if there are more rows, false if scanner is done
*/
public synchronized boolean next(List<KeyValue> result) throws IOException {
public synchronized boolean next(List<KeyValue> outResult) throws IOException {
List<KeyValue> results = new ArrayList<KeyValue>();
KeyValue peeked = this.heap.peek();
if (peeked == null) {
close();
@ -136,27 +137,25 @@ class StoreScanner implements KeyValueScanner, InternalScanner, ChangedReadersOb
switch(qcode) {
case INCLUDE:
KeyValue next = this.heap.next();
result.add(next);
results.add(next);
continue;
case DONE:
// what happens if we have 0 results?
if (result.isEmpty()) {
// try the next one.
matcher.setRow(this.heap.peek().getRow());
continue;
}
if (matcher.filterEntireRow()) {
// wow, well, um, reset the result and continue.
result.clear();
matcher.setRow(heap.peek().getRow());
continue;
// nuke all results, and then return.
results.clear();
}
// copy jazz
outResult.addAll(results);
return true;
case DONE_SCAN:
close();
// copy jazz
outResult.addAll(results);
return false;
case SEEK_NEXT_ROW:
@ -178,9 +177,14 @@ class StoreScanner implements KeyValueScanner, InternalScanner, ChangedReadersOb
throw new RuntimeException("UNEXPECTED");
}
}
if(result.size() > 0) {
if (!results.isEmpty()) {
// copy jazz
outResult.addAll(results);
return true;
}
// No more keys
close();
return false;

70
src/java/org/apache/hadoop/hbase/thrift/ThriftServer.java
View File

@ -298,12 +298,15 @@ public class ThriftServer {
HTable table = getTable(tableName);
Get get = new Get(row);
get.addColumn(family, qualifier);
get.setTimeStamp(timestamp);
get.setTimeRange(Long.MIN_VALUE, timestamp);
get.setMaxVersions(numVersions);
Result result = table.get(get);
List<Cell> cells = new ArrayList<Cell>();
for(KeyValue kv : result.sorted()) {
cells.add(new Cell(kv.getValue(), kv.getTimestamp()));
KeyValue [] kvs = result.sorted();
if (kvs != null) {
for(KeyValue kv : kvs) {
cells.add(new Cell(kv.getValue(), kv.getTimestamp()));
}
}
return ThriftUtilities.cellFromHBase(cells.toArray(new Cell[0]));
} catch (IOException e) {
@ -335,7 +338,7 @@ public class ThriftServer {
HTable table = getTable(tableName);
if (columns == null) {
Get get = new Get(row);
get.setTimeStamp(timestamp);
get.setTimeRange(Long.MIN_VALUE, timestamp);
Result result = table.get(get);
return ThriftUtilities.rowResultFromHBase(result.getRowResult());
}
@ -345,7 +348,7 @@ public class ThriftServer {
byte [][] famAndQf = KeyValue.parseColumn(column);
get.addColumn(famAndQf[0], famAndQf[1]);
}
get.setTimeStamp(timestamp);
get.setTimeRange(Long.MIN_VALUE, timestamp);
Result result = table.get(get);
return ThriftUtilities.rowResultFromHBase(result.getRowResult());
} catch (IOException e) {
@ -362,12 +365,12 @@ public class ThriftServer {
long timestamp) throws IOError {
try {
HTable table = getTable(tableName);
Delete delete = new Delete(row, timestamp, null);
Delete delete = new Delete(row);
byte [][] famAndQf = KeyValue.parseColumn(column);
if(famAndQf[1].length == 0){
delete.deleteFamily(famAndQf[0]);
if (famAndQf[1].length == 0) {
delete.deleteFamily(famAndQf[0], timestamp);
} else {
delete.deleteColumns(famAndQf[0], famAndQf[1]);
delete.deleteColumns(famAndQf[0], famAndQf[1], timestamp);
}
table.delete(delete);
@ -457,23 +460,36 @@ public class ThriftServer {
public void mutateRowsTs(byte[] tableName, List<BatchMutation> rowBatches, long timestamp)
throws IOError, IllegalArgument, TException {
List<Put> puts = new ArrayList<Put>();
List<Delete> deletes = new ArrayList<Delete>();
for (BatchMutation batch : rowBatches) {
byte[] row = batch.row;
List<Mutation> mutations = batch.mutations;
Delete delete = new Delete(row);
Put put = new Put(row);
put.setTimeStamp(timestamp);
for (Mutation m : mutations) {
byte [][] famAndQf = KeyValue.parseColumn(m.column);
put.add(famAndQf[0], famAndQf[1], m.value);
if (m.isDelete) {
delete.deleteColumns(famAndQf[0], famAndQf[1]);
} else {
put.add(famAndQf[0], famAndQf[1], m.value);
}
}
puts.add(put);
if (!delete.isEmpty())
deletes.add(delete);
if (!put.isEmpty())
puts.add(put);
}
HTable table = null;
try {
table = getTable(tableName);
table.put(puts);
if (!puts.isEmpty())
table.put(puts);
for (Delete del : deletes) {
table.delete(del);
}
} catch (IOException e) {
throw new IOError(e.getMessage());
} catch (IllegalArgumentException e) {
@ -534,19 +550,19 @@ public class ThriftServer {
public int scannerOpen(byte[] tableName, byte[] startRow,
List<byte[]> columns) throws IOError {
try {
HTable table = getTable(tableName);
byte[][] columnsArray = null;
if ((columns == null) || (columns.size() == 0)) {
columnsArray = getAllColumns(table);
} else {
columnsArray = columns.toArray(new byte[0][]);
HTable table = getTable(tableName);
byte[][] columnsArray = null;
if ((columns == null) || (columns.size() == 0)) {
columnsArray = getAllColumns(table);
} else {
columnsArray = columns.toArray(new byte[0][]);
}
Scan scan = new Scan(startRow);
scan.addColumns(columnsArray);
return addScanner(table.getScanner(scan));
} catch (IOException e) {
throw new IOError(e.getMessage());
}
Scan scan = new Scan(startRow);
scan.addColumns(columnsArray);
return addScanner(table.getScanner(scan));
} catch (IOException e) {
throw new IOError(e.getMessage());
}
}
public int scannerOpenWithStop(byte[] tableName, byte[] startRow,
@ -579,7 +595,7 @@ public class ThriftServer {
}
Scan scan = new Scan(startRow);
scan.addColumns(columnsArray);
scan.setTimeRange(0, timestamp);
scan.setTimeRange(Long.MIN_VALUE, timestamp);
return addScanner(table.getScanner(scan));
} catch (IOException e) {
throw new IOError(e.getMessage());
@ -599,7 +615,7 @@ public class ThriftServer {
}
Scan scan = new Scan(startRow, stopRow);
scan.addColumns(columnsArray);
scan.setTimeRange(0, timestamp);
scan.setTimeRange(Long.MIN_VALUE, timestamp);
return addScanner(table.getScanner(scan));
} catch (IOException e) {
throw new IOError(e.getMessage());

39
src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java
View File

@ -285,10 +285,45 @@ public class TestHRegion extends HBaseTestCase {
get = new Get(row).addColumn(fam, splitB);
result = region.get(get, null);
assertEquals(1, result.size());
}
public void testScanner_DeleteOneFamilyNotAnother() throws IOException {
byte [] tableName = Bytes.toBytes("test_table");
byte [] fam1 = Bytes.toBytes("columnA");
byte [] fam2 = Bytes.toBytes("columnB");
initHRegion(tableName, getName(), fam1, fam2);
byte [] rowA = Bytes.toBytes("rowA");
byte [] rowB = Bytes.toBytes("rowB");
byte [] value = Bytes.toBytes("value");
Delete delete = new Delete(rowA);
delete.deleteFamily(fam1);
region.delete(delete, null, true);
// now create data.
Put put = new Put(rowA);
put.add(fam2, null, value);
region.put(put);
put = new Put(rowB);
put.add(fam1, null, value);
put.add(fam2, null, value);
region.put(put);
Scan scan = new Scan();
scan.addFamily(fam1).addFamily(fam2);
InternalScanner s = region.getScanner(scan);
List<KeyValue> results = new ArrayList<KeyValue>();
s.next(results);
assertTrue(Bytes.equals(rowA, results.get(0).getRow()));
results.clear();
s.next(results);
assertTrue(Bytes.equals(rowB, results.get(0).getRow()));
}
//Visual test, since the method doesn't return anything

6
src/test/org/apache/hadoop/hbase/regionserver/TestMemcache.java
View File

@ -506,9 +506,12 @@ public class TestMemcache extends TestCase {
KeyValue put1 = new KeyValue(row, fam, qf1, ts, val);
KeyValue put2 = new KeyValue(row, fam, qf2, ts, val);
KeyValue put3 = new KeyValue(row, fam, qf3, ts, val);
KeyValue put4 = new KeyValue(row, fam, qf3, ts+1, val);
memcache.add(put1);
memcache.add(put2);
memcache.add(put3);
memcache.add(put4);
KeyValue del =
new KeyValue(row, fam, null, ts, KeyValue.Type.DeleteFamily, val);
@ -516,8 +519,9 @@ public class TestMemcache extends TestCase {
List<KeyValue> expected = new ArrayList<KeyValue>();
expected.add(del);
expected.add(put4);
assertEquals(1, memcache.memcache.size());
assertEquals(2, memcache.memcache.size());
int i=0;
for(KeyValue actual : memcache.memcache) {
assertEquals(expected.get(i++), actual);

9
src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
View File

@ -139,9 +139,14 @@ public class TestStoreScanner extends TestCase {
getCols("a"), scanners);
List<KeyValue> results = new ArrayList<KeyValue>();
assertEquals(true, scan.next(results));
assertEquals(0, results.size());
assertEquals(true, scan.next(results));
assertEquals(1, results.size());
assertEquals(kvs[2], results.get(0));
assertEquals(false, scan.next(results));
}
public void testDeleteVersionMaskingMultiplePuts() throws IOException {
@ -272,8 +277,12 @@ public class TestStoreScanner extends TestCase {
null, scanners);
List<KeyValue> results = new ArrayList<KeyValue>();
assertEquals(true, scan.next(results));
assertEquals(0, results.size());
assertEquals(true, scan.next(results));
assertEquals(1, results.size());
assertEquals(kvs[kvs.length-1], results.get(0));
assertEquals(false, scan.next(results));
}
public void testDeleteColumn() throws IOException {

57
src/test/org/apache/hadoop/hbase/thrift/DisabledTestThriftServer.java → src/test/org/apache/hadoop/hbase/thrift/TestThriftServer.java
View File

@ -1,5 +1,5 @@
/**
* Copyright 2008-2009 The Apache Software Foundation
/*
* Copyright 2009 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@ -35,14 +35,13 @@ import org.apache.hadoop.hbase.util.Bytes;
* Unit testing for ThriftServer.HBaseHandler, a part of the
* org.apache.hadoop.hbase.thrift package.
*/
public class DisabledTestThriftServer extends HBaseClusterTestCase {
public class TestThriftServer extends HBaseClusterTestCase {
// Static names for tables, columns, rows, and values
private static byte[] tableAname = Bytes.toBytes("tableA");
private static byte[] tableBname = Bytes.toBytes("tableB");
private static byte[] columnAname = Bytes.toBytes("columnA:");
private static byte[] columnBname = Bytes.toBytes("columnB:");
private static byte[] badColumnName = Bytes.toBytes("noColon:");
private static byte[] rowAname = Bytes.toBytes("rowA");
private static byte[] rowBname = Bytes.toBytes("rowB");
private static byte[] valueAname = Bytes.toBytes("valueA");
@ -176,8 +175,6 @@ public class DisabledTestThriftServer extends HBaseClusterTestCase {
long time1 = System.currentTimeMillis();
handler.mutateRowTs(tableAname, rowAname, getMutations(), time1);
// Sleep to assure that 'time1' and 'time2' will be different even with a
// coarse grained system timer.
Thread.sleep(1000);
// Apply timestamped BatchMutations for rowA and rowB
@ -187,22 +184,25 @@ public class DisabledTestThriftServer extends HBaseClusterTestCase {
// Apply an overlapping timestamped mutation to rowB
handler.mutateRowTs(tableAname, rowBname, getMutations(), time2);
// the getVerTs is [inf, ts) so you need to increment one.
time1 += 1;
time2 += 2;
// Assert that the timestamp-related methods retrieve the correct data
assertEquals(handler.getVerTs(tableAname, rowAname, columnBname, time2,
MAXVERSIONS).size(), 2);
assertEquals(handler.getVerTs(tableAname, rowAname, columnBname, time1,
MAXVERSIONS).size(), 1);
assertEquals(2, handler.getVerTs(tableAname, rowAname, columnBname, time2,
MAXVERSIONS).size());
assertEquals(1, handler.getVerTs(tableAname, rowAname, columnBname, time1,
MAXVERSIONS).size());
TRowResult rowResult1 = handler.getRowTs(tableAname, rowAname, time1).get(0);
TRowResult rowResult2 = handler.getRowTs(tableAname, rowAname, time2).get(0);
assertTrue(Bytes.equals(rowResult1.columns.get(columnAname).value, valueAname));
// columnA was completely deleted
//assertTrue(Bytes.equals(rowResult1.columns.get(columnAname).value, valueAname));
assertTrue(Bytes.equals(rowResult1.columns.get(columnBname).value, valueBname));
assertTrue(Bytes.equals(rowResult2.columns.get(columnBname).value, valueCname));
// Maybe I'd reading this wrong but at line #187 above, the BatchMutations
// are adding a columnAname at time2 so the below should be true not false
// -- St.Ack
assertTrue(rowResult2.columns.containsKey(columnAname));
// ColumnAname has been deleted, and will never be visible even with a getRowTs()
assertFalse(rowResult2.columns.containsKey(columnAname));
List<byte[]> columns = new ArrayList<byte[]>();
columns.add(columnBname);
@ -216,14 +216,22 @@ public class DisabledTestThriftServer extends HBaseClusterTestCase {
assertFalse(rowResult1.columns.containsKey(columnAname));
// Apply some timestamped deletes
// this actually deletes _everything_.
// nukes everything in columnB: forever.
handler.deleteAllTs(tableAname, rowAname, columnBname, time1);
handler.deleteAllRowTs(tableAname, rowBname, time2);
// Assert that the timestamp-related methods retrieve the correct data
int size = handler.getVerTs(tableAname, rowAname, columnBname, time1, MAXVERSIONS).size();
assertFalse(size > 0);
assertEquals(0, size);
size = handler.getVerTs(tableAname, rowAname, columnBname, time2, MAXVERSIONS).size();
assertEquals(1, size);
// should be available....
assertTrue(Bytes.equals(handler.get(tableAname, rowAname, columnBname).get(0).value, valueCname));
assertFalse(handler.getRow(tableAname, rowBname).size() > 0);
assertEquals(0, handler.getRow(tableAname, rowBname).size());
// Teardown
handler.disableTable(tableAname);
@ -253,6 +261,9 @@ public class DisabledTestThriftServer extends HBaseClusterTestCase {
long time2 = System.currentTimeMillis();
handler.mutateRowsTs(tableAname, getBatchMutations(), time2);
time1 += 1;
time2 += 1;
// Test a scanner on all rows and all columns, no timestamp
int scanner1 = handler.scannerOpen(tableAname, rowAname, getColumnList(true, true));
TRowResult rowResult1a = handler.scannerGet(scanner1).get(0);
@ -260,8 +271,9 @@ public class DisabledTestThriftServer extends HBaseClusterTestCase {
// This used to be '1'. I don't know why when we are asking for two columns
// and when the mutations above would seem to add two columns to the row.
// -- St.Ack 05/12/2009
assertEquals(rowResult1a.columns.size(), 2);
assertEquals(rowResult1a.columns.size(), 1);
assertTrue(Bytes.equals(rowResult1a.columns.get(columnBname).value, valueCname));
TRowResult rowResult1b = handler.scannerGet(scanner1).get(0);
assertTrue(Bytes.equals(rowResult1b.row, rowBname));
assertEquals(rowResult1b.columns.size(), 2);
@ -272,8 +284,9 @@ public class DisabledTestThriftServer extends HBaseClusterTestCase {
// Test a scanner on all rows and all columns, with timestamp
int scanner2 = handler.scannerOpenTs(tableAname, rowAname, getColumnList(true, true), time1);
TRowResult rowResult2a = handler.scannerGet(scanner2).get(0);
assertEquals(rowResult2a.columns.size(), 2);
assertTrue(Bytes.equals(rowResult2a.columns.get(columnAname).value, valueAname));
assertEquals(rowResult2a.columns.size(), 1);
// column A deleted, does not exist.
//assertTrue(Bytes.equals(rowResult2a.columns.get(columnAname).value, valueAname));
assertTrue(Bytes.equals(rowResult2a.columns.get(columnBname).value, valueBname));
closeScanner(scanner2, handler);
@ -350,18 +363,22 @@ public class DisabledTestThriftServer extends HBaseClusterTestCase {
*/
private List<BatchMutation> getBatchMutations() {
List<BatchMutation> batchMutations = new ArrayList<BatchMutation>();
// Mutations to rowA. You can't mix delete and put anymore.
List<Mutation> rowAmutations = new ArrayList<Mutation>();
rowAmutations.add(new Mutation(true, columnAname, null));
batchMutations.add(new BatchMutation(rowAname, rowAmutations));
rowAmutations = new ArrayList<Mutation>();
rowAmutations.add(new Mutation(false, columnBname, valueCname));
batchMutations.add(new BatchMutation(rowAname, rowAmutations));
// Mutations to rowB
List<Mutation> rowBmutations = new ArrayList<Mutation>();
rowBmutations.add(new Mutation(false, columnAname, valueCname));
rowBmutations.add(new Mutation(false, columnBname, valueDname));
batchMutations.add(new BatchMutation(rowBname, rowBmutations));
return batchMutations;
}