From 60079f0cd08a65abd21f5ee3e1d863dbb24419fb Mon Sep 17 00:00:00 2001 From: sershe Date: Thu, 3 Apr 2014 20:29:30 +0000 Subject: [PATCH] HBASE-10118 Major compact keeps deletes with future timestamps (Liu Shaohui) git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1584383 13f79535-47bb-0310-9956-ffa450edef68 --- .../hbase/regionserver/ScanQueryMatcher.java | 9 ++- .../hbase/client/TestFromClientSide.java | 61 +++++++++++++++++++ src/main/docbkx/book.xml | 9 +++ 3 files changed, 76 insertions(+), 3 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java index 92eefd8e233..7ba5e6e2083 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java @@ -336,9 +336,12 @@ public class ScanQueryMatcher { } // Can't early out now, because DelFam come before any other keys } - if (retainDeletesInOutput - || (!isUserScan && (EnvironmentEdgeManager.currentTimeMillis() - timestamp) <= timeToPurgeDeletes) - || kv.getMvccVersion() > maxReadPointToTrackVersions) { + + if ((!isUserScan) + && timeToPurgeDeletes > 0 + && (EnvironmentEdgeManager.currentTimeMillis() - timestamp) <= timeToPurgeDeletes) { + return MatchCode.INCLUDE; + } else if (retainDeletesInOutput || kv.getMvccVersion() > maxReadPointToTrackVersions) { // always include or it is not time yet to check whether it is OK // to purge deltes or not if (!isUserScan) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java index 428c73369c0..02fa62734ca 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java @@ -62,6 +62,7 @@ import org.apache.hadoop.hbase.LargeTests; import org.apache.hadoop.hbase.MiniHBaseCluster; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.Waiter; import org.apache.hadoop.hbase.client.metrics.ScanMetrics; import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint; @@ -83,6 +84,7 @@ import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel; import org.apache.hadoop.hbase.ipc.RpcClient; import org.apache.hadoop.hbase.ipc.RpcServer; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.protobuf.generated.AdminProtos; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType; import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MultiRowMutationService; @@ -227,6 +229,65 @@ public class TestFromClientSide { h.close(); } + /** + * Basic client side validation of HBASE-10118 + */ + @Test + public void testPurgeFutureDeletes() throws Exception { + final byte[] TABLENAME = Bytes.toBytes("testPurgeFutureDeletes"); + final byte[] ROW = Bytes.toBytes("row"); + final byte[] FAMILY = Bytes.toBytes("family"); + final byte[] COLUMN = Bytes.toBytes("column"); + final byte[] VALUE = Bytes.toBytes("value"); + + HTable table = TEST_UTIL.createTable(TABLENAME, FAMILY); + + // future timestamp + long ts = System.currentTimeMillis() * 2; + Put put = new Put(ROW, ts); + put.add(FAMILY, COLUMN, VALUE); + table.put(put); + + Get get = new Get(ROW); + Result result = table.get(get); + assertArrayEquals(VALUE, result.getValue(FAMILY, COLUMN)); + + Delete del = new Delete(ROW); + del.deleteColumn(FAMILY, COLUMN, ts); + table.delete(del); + + get = new Get(ROW); + result = table.get(get); + assertNull(result.getValue(FAMILY, COLUMN)); + + // major compaction, purged future deletes + TEST_UTIL.getHBaseAdmin().flush(TABLENAME); + TEST_UTIL.getHBaseAdmin().majorCompact(TABLENAME); + + // waiting for the major compaction to complete + TEST_UTIL.waitFor(6000, new Waiter.Predicate() { + @Override + public boolean evaluate() throws IOException { + try { + return TEST_UTIL.getHBaseAdmin().getCompactionState(TABLENAME) == + AdminProtos.GetRegionInfoResponse.CompactionState.NONE; + } catch (InterruptedException e) { + throw new IOException(e); + } + } + }); + + put = new Put(ROW, ts); + put.add(FAMILY, COLUMN, VALUE); + table.put(put); + + get = new Get(ROW); + result = table.get(get); + assertArrayEquals(VALUE, result.getValue(FAMILY, COLUMN)); + + table.close(); + } + @Test public void testSharedZooKeeper() throws Exception { Configuration newConfig = new Configuration(TEST_UTIL.getConfiguration()); diff --git a/src/main/docbkx/book.xml b/src/main/docbkx/book.xml index 1747d0f0174..023bd2b3dd1 100644 --- a/src/main/docbkx/book.xml +++ b/src/main/docbkx/book.xml @@ -568,6 +568,15 @@ htable.put(put); up on the user mailing list. Also see for more information on the internal KeyValue format. + Delete markers are purged during the major compaction of store, + unless the KEEP_DELETED_CELLS is set in the column family. In some + scenarios, users want to keep the deletes for a time and you can set the + delete TTL: hbase.hstore.time.to.purge.deletes in the configuration. + If this delete TTL is not set, or set to 0, all delete markers including those + with future timestamp are purged during the later major compaction. + Otherwise, a delete marker is kept until the major compaction after + marker's timestamp + delete TTL. +