diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java index 04a08bc8494..c0a4b51019f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java @@ -292,6 +292,8 @@ public class DefaultMemStore implements MemStore { KeyValue found = this.snapshot.get(kv); if (found != null && found.getMvccVersion() == kv.getMvccVersion()) { this.snapshot.remove(kv); + long sz = heapSizeChange(kv, true); + this.snapshotSize -= sz; } // If the key is in the memstore, delete it. Update this.size. found = this.kvset.get(kv); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java index 2334e8df28c..1edcba40601 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java @@ -116,6 +116,7 @@ import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl; import org.apache.hadoop.hbase.regionserver.HRegion.RowLock; import org.apache.hadoop.hbase.regionserver.TestStore.FaultyFileSystem; +import org.apache.hadoop.hbase.regionserver.wal.FaultyHLog; import org.apache.hadoop.hbase.regionserver.wal.HLog; import org.apache.hadoop.hbase.regionserver.wal.HLogFactory; import org.apache.hadoop.hbase.regionserver.wal.HLogKey; @@ -234,6 +235,58 @@ public class TestHRegion { HRegion.closeHRegion(region); } + /* + * This test is for verifying memstore snapshot size is correctly updated in case of rollback + * See HBASE-10845 + */ + @Test (timeout=60000) + public void testMemstoreSnapshotSize() throws IOException { + class MyFaultyHLog extends FaultyHLog { + StoreFlushContext storeFlushCtx; + public MyFaultyHLog(FileSystem fs, Path rootDir, String logName, Configuration conf) + throws IOException { + super(fs, rootDir, logName, conf); + } + + void setStoreFlushCtx(StoreFlushContext storeFlushCtx) { + this.storeFlushCtx = storeFlushCtx; + } + + @Override + public void sync(long txid) throws IOException { + storeFlushCtx.prepare(); + super.sync(txid); + } + } + + FileSystem fs = FileSystem.get(CONF); + Path rootDir = new Path(dir + "testMemstoreSnapshotSize"); + MyFaultyHLog faultyLog = new MyFaultyHLog(fs, rootDir, "testMemstoreSnapshotSize", CONF); + HRegion region = initHRegion(tableName, null, null, name.getMethodName(), + CONF, false, Durability.SYNC_WAL, faultyLog, COLUMN_FAMILY_BYTES); + + Store store = region.getStore(COLUMN_FAMILY_BYTES); + // Get some random bytes. + byte [] value = Bytes.toBytes(name.getMethodName()); + faultyLog.setStoreFlushCtx(store.createFlushContext(12345)); + + Put put = new Put(value); + put.add(COLUMN_FAMILY_BYTES, Bytes.toBytes("abc"), value); + faultyLog.setFailureType(FaultyHLog.FailureType.SYNC); + + boolean threwIOE = false; + try { + region.put(put); + } catch (IOException ioe) { + threwIOE = true; + } finally { + assertTrue("The regionserver should have thrown an exception", threwIOE); + } + long sz = store.getFlushableSize(); + assertTrue("flushable size should be zero, but it is " + sz, sz == 0); + HRegion.closeHRegion(region); + } + /** * Test we do not lose data if we fail a flush and then close. * Part of HBase-10466. Tests the following from the issue description: diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultyHLog.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultyHLog.java new file mode 100644 index 00000000000..10ba82faaec --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/FaultyHLog.java @@ -0,0 +1,70 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.wal; + +import java.io.IOException; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; + +/* + * This is a utility class, used by tests, which fails operation specified by FailureType enum + */ +public class FaultyHLog extends FSHLog { + public enum FailureType { + NONE, APPENDNOSYNC, SYNC + } + FailureType ft = FailureType.NONE; + + public FaultyHLog(FileSystem fs, Path rootDir, String logName, Configuration conf) + throws IOException { + super(fs, rootDir, logName, conf); + } + + public void setFailureType(FailureType fType) { + this.ft = fType; + } + + @Override + public void sync(long txid) throws IOException { + if (this.ft == FailureType.SYNC) { + throw new IOException("sync"); + } + super.sync(txid); + } + @Override + public long appendNoSync(HRegionInfo info, TableName tableName, WALEdit edits, + List clusterIds, final long now, HTableDescriptor htd, AtomicLong sequenceId, + boolean isInMemstore, long nonceGroup, long nonce) throws IOException { + if (this.ft == FailureType.APPENDNOSYNC) { + throw new IOException("appendNoSync"); + } + return super.appendNoSync(info, tableName, edits, clusterIds, now, htd, sequenceId, + isInMemstore, nonceGroup, nonce); + } +} +