Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-14721 Memstore add cells - Avoid many garbage.

This commit is contained in:
anoopsjohn 2015-10-30 18:39:42 +05:30
parent 84f4cceb48
commit 23fa18184c
6 changed files with 23 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
View File

@ -45,7 +45,6 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.ClassSize;
import org.apache.hadoop.hbase.util.CollectionBackedScanner;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.ReflectionUtils;
import org.apache.htrace.Trace;
@ -221,13 +220,12 @@ public class DefaultMemStore implements MemStore {
/**
* Write an update
* @param cell
* @return approximate size of the passed KV & newly added KV which maybe different than the
* passed-in KV
* @return approximate size of the passed Cell.
*/
@Override
public Pair<Long, Cell> add(Cell cell) {
public long add(Cell cell) {
Cell toAdd = maybeCloneWithAllocator(cell);
return new Pair<Long, Cell>(internalAdd(toAdd), toAdd);
return internalAdd(toAdd);
}
@Override
@ -964,21 +962,17 @@ public class DefaultMemStore implements MemStore {
byte [] empty = new byte[0];
for (int i = 0; i < count; i++) {
// Give each its own ts
Pair<Long, Cell> ret = memstore1.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, empty));
size += ret.getFirst();
size += memstore1.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, empty));
}
LOG.info("memstore1 estimated size=" + size);
for (int i = 0; i < count; i++) {
Pair<Long, Cell> ret = memstore1.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, empty));
size += ret.getFirst();
size += memstore1.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, empty));
}
LOG.info("memstore1 estimated size (2nd loading of same data)=" + size);
// Make a variably sized memstore.
DefaultMemStore memstore2 = new DefaultMemStore();
for (int i = 0; i < count; i++) {
Pair<Long, Cell> ret = memstore2.add(new KeyValue(Bytes.toBytes(i), fam, qf, i,
new byte[i]));
size += ret.getFirst();
size += memstore2.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, new byte[i]));
}
LOG.info("memstore2 estimated size=" + size);
final int seconds = 30;

20
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
View File

@ -3686,15 +3686,10 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
int listSize = cells.size();
for (int i=0; i < listSize; i++) {
Cell cell = cells.get(i);
if (cell.getSequenceId() == 0) {
if (cell.getSequenceId() == 0 || isInReplay) {
CellUtil.setSequenceId(cell, mvccNum);
}
Pair<Long, Cell> ret = store.add(cell);
size += ret.getFirst();
if(isInReplay) {
// set memstore newly added cells with replay mvcc number
CellUtil.setSequenceId(ret.getSecond(), mvccNum);
}
size += store.add(cell);
}
}
@ -4937,7 +4932,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
* @return True if we should flush.
*/
protected boolean restoreEdit(final Store s, final Cell cell) {
long kvSize = s.add(cell).getFirst();
long kvSize = s.add(cell);
if (this.rsAccounting != null) {
rsAccounting.addAndGetRegionReplayEditsSize(getRegionInfo().getRegionName(), kvSize);
}
@ -6829,8 +6824,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
checkFamily(CellUtil.cloneFamily(cell));
// unreachable
}
Pair<Long, Cell> ret = store.add(cell);
addedSize += ret.getFirst();
addedSize += store.add(cell);
}
}
@ -7168,8 +7162,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
// otherwise keep older versions around
for (Cell cell: entry.getValue()) {
CellUtil.setSequenceId(cell, writeEntry.getWriteNumber());
Pair<Long, Cell> ret = store.add(cell);
size += ret.getFirst();
size += store.add(cell);
doRollBackMemstore = true;
}
}
@ -7399,8 +7392,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
// otherwise keep older versions around
for (Cell cell : entry.getValue()) {
CellUtil.setSequenceId(cell, writeEntry.getWriteNumber());
Pair<Long, Cell> ret = store.add(cell);
size += ret.getFirst();
size += store.add(cell);
doRollBackMemstore = true;
}
}

3
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
View File

@ -89,7 +89,6 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.ChecksumType;
import org.apache.hadoop.hbase.util.ClassSize;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.ReflectionUtils;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.util.StringUtils;
@ -675,7 +674,7 @@ public class HStore implements Store {
}
@Override
public Pair<Long, Cell> add(final Cell cell) {
public long add(final Cell cell) {
lock.readLock().lock();
try {
return this.memstore.add(cell);

6
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
View File

@ -22,7 +22,6 @@ import java.util.List;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.io.HeapSize;
import org.apache.hadoop.hbase.util.Pair;
/**
* The MemStore holds in-memory modifications to the Store. Modifications are {@link Cell}s.
@ -68,10 +67,9 @@ public interface MemStore extends HeapSize {
/**
* Write an update
* @param cell
* @return approximate size of the passed KV and the newly added KV which maybe different from the
* passed in KV.
* @return approximate size of the passed cell.
*/
Pair<Long, Cell> add(final Cell cell);
long add(final Cell cell);
/**
* @return Oldest timestamp of all the Cells in the MemStore

5
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
View File

@ -44,7 +44,6 @@ import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionThroughputController;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.util.Pair;
/**
* Interface for objects that hold a column family in a Region. Its a memstore and a set of zero or
@ -127,9 +126,9 @@ public interface Store extends HeapSize, StoreConfigInformation, PropagatingConf
/**
* Adds a value to the memstore
* @param cell
* @return memstore size delta &amp; newly added KV which maybe different than the passed in KV
* @return memstore size delta
*/
Pair<Long, Cell> add(Cell cell);
long add(Cell cell);
/**
* When was the last edit done in the memstore

17
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
View File

@ -223,7 +223,7 @@ public class TestStore {
long size = store.memstore.getFlushableSize();
Assert.assertEquals(0, size);
LOG.info("Adding some data");
long kvSize = store.add(new KeyValue(row, family, qf1, 1, (byte[])null)).getFirst();
long kvSize = store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
size = store.memstore.getFlushableSize();
Assert.assertEquals(kvSize, size);
// Flush. Bug #1 from HBASE-10466. Make sure size calculation on failed flush is right.
@ -638,20 +638,15 @@ public class TestStore {
size += this.store.add(new KeyValue(Bytes.toBytes("200909091000"), family, qf1,
System.currentTimeMillis(),
Bytes.toBytes(newValue))).getFirst();
System.currentTimeMillis(), Bytes.toBytes(newValue)));
size += this.store.add(new KeyValue(Bytes.toBytes("200909091200"), family, qf1,
System.currentTimeMillis(),
Bytes.toBytes(newValue))).getFirst();
System.currentTimeMillis(), Bytes.toBytes(newValue)));
size += this.store.add(new KeyValue(Bytes.toBytes("200909091300"), family, qf1,
System.currentTimeMillis(),
Bytes.toBytes(newValue))).getFirst();
System.currentTimeMillis(), Bytes.toBytes(newValue)));
size += this.store.add(new KeyValue(Bytes.toBytes("200909091400"), family, qf1,
System.currentTimeMillis(),
Bytes.toBytes(newValue))).getFirst();
System.currentTimeMillis(), Bytes.toBytes(newValue)));
size += this.store.add(new KeyValue(Bytes.toBytes("200909091500"), family, qf1,
System.currentTimeMillis(),
Bytes.toBytes(newValue))).getFirst();
System.currentTimeMillis(), Bytes.toBytes(newValue)));
for ( int i = 0 ; i < 10000 ; ++i) {