From 80a9bf0eb6c47a1ec14c888a0d8950e08a906c7e Mon Sep 17 00:00:00 2001 From: anastas Date: Sun, 5 Nov 2017 15:29:48 +0200 Subject: [PATCH] HBASE-18232 Jumbo Chunks support for the future CellChunkMap index usage --- .../hadoop/hbase/regionserver/Chunk.java | 4 + .../hbase/regionserver/ChunkCreator.java | 69 ++++++++++++---- .../regionserver/ImmutableMemStoreLAB.java | 25 +++++- .../hbase/regionserver/MemStoreLAB.java | 22 +++++- .../hbase/regionserver/MemStoreLABImpl.java | 20 +++++ .../hbase/regionserver/TestCellFlatSet.java | 78 +++++++++++++++---- .../regionserver/TestCompactingMemStore.java | 2 +- .../TestCompactingToCellFlatMapMemStore.java | 23 +++--- 8 files changed, 195 insertions(+), 48 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Chunk.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Chunk.java index 61cd591a4e9..6c3f471d867 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Chunk.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Chunk.java @@ -85,6 +85,10 @@ public abstract class Chunk { return this.fromPool; } + boolean isJumbo() { + return size > ChunkCreator.getInstance().getChunkSize(); + } + /** * Actually claim the memory for this chunk. This should only be called from the thread that * constructed the chunk. It is thread-safe against other threads calling alloc(), who will block diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChunkCreator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChunkCreator.java index 551fdb10e82..89317c9bce5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChunkCreator.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChunkCreator.java @@ -111,17 +111,27 @@ public class ChunkCreator { * @return the chunk that was initialized */ Chunk getChunk() { - return getChunk(CompactingMemStore.IndexType.ARRAY_MAP); + return getChunk(CompactingMemStore.IndexType.ARRAY_MAP, chunkSize); + } + + /** + * Creates and inits a chunk. The default implementation for specific index type. + * @return the chunk that was initialized + */ + Chunk getChunk(CompactingMemStore.IndexType chunkIndexType) { + return getChunk(chunkIndexType, chunkSize); } /** * Creates and inits a chunk. * @return the chunk that was initialized * @param chunkIndexType whether the requested chunk is going to be used with CellChunkMap index + * @param size the size of the chunk to be allocated, in bytes */ - Chunk getChunk(CompactingMemStore.IndexType chunkIndexType) { + Chunk getChunk(CompactingMemStore.IndexType chunkIndexType, int size) { Chunk chunk = null; - if (pool != null) { + // if we have pool and this is not jumbo chunk (when size != chunkSize this is jumbo chunk) + if ((pool != null) && (size == chunkSize)) { // the pool creates the chunk internally. The chunk#init() call happens here chunk = this.pool.getChunk(); // the pool has run out of maxCount @@ -133,9 +143,9 @@ public class ChunkCreator { } } if (chunk == null) { - // the second boolean parameter means: - // if CellChunkMap index is requested, put allocated on demand chunk mapping into chunkIdMap - chunk = createChunk(false, chunkIndexType); + // the second parameter explains whether CellChunkMap index is requested, + // in that case, put allocated on demand chunk mapping into chunkIdMap + chunk = createChunk(false, chunkIndexType, size); } // now we need to actually do the expensive memory allocation step in case of a new chunk, @@ -144,21 +154,38 @@ public class ChunkCreator { return chunk; } + /** + * Creates and inits a chunk of a special size, bigger than a regular chunk size. + * Such a chunk will never come from pool and will always be on demand allocated. + * @return the chunk that was initialized + * @param chunkIndexType whether the requested chunk is going to be used with CellChunkMap index + * @param jumboSize the special size to be used + */ + Chunk getJumboChunk(CompactingMemStore.IndexType chunkIndexType, int jumboSize) { + if (jumboSize <= chunkSize) { + LOG.warn("Jumbo chunk size " + jumboSize + " must be more than regular chunk size " + + chunkSize + ". Converting to regular chunk."); + getChunk(chunkIndexType,chunkSize); + } + return getChunk(chunkIndexType, jumboSize); + } + /** * Creates the chunk either onheap or offheap * @param pool indicates if the chunks have to be created which will be used by the Pool - * @param chunkIndexType + * @param chunkIndexType whether the requested chunk is going to be used with CellChunkMap index + * @param size the size of the chunk to be allocated, in bytes * @return the chunk */ - private Chunk createChunk(boolean pool, CompactingMemStore.IndexType chunkIndexType) { + private Chunk createChunk(boolean pool, CompactingMemStore.IndexType chunkIndexType, int size) { Chunk chunk = null; int id = chunkID.getAndIncrement(); assert id > 0; // do not create offheap chunk on demand if (pool && this.offheap) { - chunk = new OffheapChunk(chunkSize, id, pool); + chunk = new OffheapChunk(size, id, pool); } else { - chunk = new OnheapChunk(chunkSize, id, pool); + chunk = new OnheapChunk(size, id, pool); } if (pool || (chunkIndexType == CompactingMemStore.IndexType.CHUNK_MAP)) { // put the pool chunk into the chunkIdMap so it is not GC-ed @@ -167,6 +194,12 @@ public class ChunkCreator { return chunk; } + // Chunks from pool are created covered with strong references anyway + // TODO: change to CHUNK_MAP if it is generally defined + private Chunk createChunkForPool() { + return createChunk(true, CompactingMemStore.IndexType.ARRAY_MAP, chunkSize); + } + @VisibleForTesting // Used to translate the ChunkID into a chunk ref Chunk getChunk(int id) { @@ -228,9 +261,7 @@ public class ChunkCreator { this.poolSizePercentage = poolSizePercentage; this.reclaimedChunks = new LinkedBlockingQueue<>(); for (int i = 0; i < initialCount; i++) { - // Chunks from pool are covered with strong references anyway - // TODO: change to CHUNK_MAP if it is generally defined - Chunk chunk = createChunk(true, CompactingMemStore.IndexType.ARRAY_MAP); + Chunk chunk = createChunkForPool(); chunk.init(); reclaimedChunks.add(chunk); } @@ -262,8 +293,7 @@ public class ChunkCreator { long created = this.chunkCount.get(); if (created < this.maxCount) { if (this.chunkCount.compareAndSet(created, created + 1)) { - // TODO: change to CHUNK_MAP if it is generally defined - chunk = createChunk(true, CompactingMemStore.IndexType.ARRAY_MAP); + chunk = createChunkForPool(); break; } } else { @@ -434,7 +464,14 @@ public class ChunkCreator { // this translation will (most likely) return null Chunk chunk = ChunkCreator.this.getChunk(chunkID); if (chunk != null) { - pool.putbackChunks(chunk); + // Jumbo chunks are covered with chunkIdMap, but are not from pool, so such a chunk should + // be released here without going to pool. + // Removing them from chunkIdMap will cause their removal by the GC. + if (chunk.isJumbo()) { + this.removeChunk(chunkID); + } else { + pool.putbackChunks(chunk); + } } // if chunk is null, it was never covered by the chunkIdMap (and so wasn't in pool also), // so we have nothing to do on its release diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java index b035f1b2d72..b2c91300f14 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java @@ -45,15 +45,34 @@ public class ImmutableMemStoreLAB implements MemStoreLAB { throw new IllegalStateException("This is an Immutable MemStoreLAB."); } + /* Creating chunk to be used as index chunk in CellChunkMap, part of the chunks array. + ** Returning a new chunk, without replacing current chunk, + ** meaning MSLABImpl does not make the returned chunk as CurChunk. + ** The space on this chunk will be allocated externally. + ** The interface is only for external callers + */ @Override - // returning a new chunk, without replacing current chunk, - // the space on this chunk will be allocated externally - // use the first MemStoreLABImpl in the list public Chunk getNewExternalChunk() { MemStoreLAB mslab = this.mslabs.get(0); return mslab.getNewExternalChunk(); } + /* Creating chunk to be used as data chunk in CellChunkMap. + ** This chunk is bigger the normal constant chunk size, and thus called JumboChunk it is used for + ** jumbo cells (which size is bigger than normal chunks). + ** Jumbo Chunks are needed only for CCM and thus are created only in + ** CompactingMemStore.IndexType.CHUNK_MAP type. + ** Returning a new chunk, without replacing current chunk, + ** meaning MSLABImpl does not make the returned chunk as CurChunk. + ** The space on this chunk will be allocated externally. + ** The interface is only for external callers + */ + @Override + public Chunk getNewExternalJumboChunk(int size) { + MemStoreLAB mslab = this.mslabs.get(0); + return mslab.getNewExternalJumboChunk(size); + } + @Override public void close() { // 'openScannerCount' here tracks the scanners opened on segments which directly refer to this diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java index 24865e63cbf..1982b4fb4e2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java @@ -83,12 +83,26 @@ public interface MemStoreLAB { */ void decScannerCount(); - /** - * Return a new empty chunk without considering this chunk as current - * The space on this chunk will be allocated externally - */ + /* Creating chunk to be used as index chunk in CellChunkMap, part of the chunks array. + ** Returning a new chunk, without replacing current chunk, + ** meaning MSLABImpl does not make the returned chunk as CurChunk. + ** The space on this chunk will be allocated externally. + ** The interface is only for external callers + */ Chunk getNewExternalChunk(); + /* Creating chunk to be used as data chunk in CellChunkMap. + ** This chunk is bigger the normal constant chunk size, and thus called JumboChunk it is used for + ** jumbo cells (which size is bigger than normal chunks). + ** Jumbo Chunks are needed only for CCM and thus are created only in + ** CompactingMemStore.IndexType.CHUNK_MAP type. + ** Returning a new chunk, without replacing current chunk, + ** meaning MSLABImpl does not make the returned chunk as CurChunk. + ** The space on this chunk will be allocated externally. + ** The interface is only for external callers + */ + Chunk getNewExternalJumboChunk(int size); + public static MemStoreLAB newInstance(Configuration conf) { MemStoreLAB memStoreLAB = null; if (isEnabled(conf)) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java index c7035eda515..38f79533a5f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java @@ -264,6 +264,26 @@ public class MemStoreLABImpl implements MemStoreLAB { return c; } + /* Creating chunk to be used as data chunk in CellChunkMap. + ** This chunk is bigger than normal constant chunk size, and thus called JumboChunk. + ** JumboChunk is used for jumbo cell (which size is bigger than normal chunk). It is allocated + ** once per cell. So even if there is space this is not reused. + ** Jumbo Chunks are used only for CCM and thus are created only in + ** CompactingMemStore.IndexType.CHUNK_MAP type. + ** Returning a new chunk, without replacing current chunk, + ** meaning MSLABImpl does not make the returned chunk as CurChunk. + ** The space on this chunk will be allocated externally. + ** The interface is only for external callers + */ + @Override + public Chunk getNewExternalJumboChunk(int size) { + // the new chunk is going to hold the jumbo cell data and need to be referenced by a strong map + // thus giving the CCM index type + Chunk c = this.chunkCreator.getJumboChunk(CompactingMemStore.IndexType.CHUNK_MAP, size); + chunks.add(c.getId()); + return c; + } + @VisibleForTesting Chunk getCurrentChunk() { return this.curChunk.get(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellFlatSet.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellFlatSet.java index cf425b0a89a..f2a4220d819 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellFlatSet.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellFlatSet.java @@ -58,6 +58,7 @@ public class TestCellFlatSet extends TestCase { return new Object[] { "SMALL_CHUNKS", "NORMAL_CHUNKS" }; // test with different chunk sizes } private static final int NUM_OF_CELLS = 4; + private static final int SMALL_CHUNK_SIZE = 64; private Cell ascCells[]; private CellArrayMap ascCbOnHeap; private Cell descCells[]; @@ -69,8 +70,7 @@ public class TestCellFlatSet extends TestCase { private CellChunkMap ascCCM; // for testing ascending CellChunkMap with one chunk in array private CellChunkMap descCCM; // for testing descending CellChunkMap with one chunk in array - private CellChunkMap ascMultCCM; // testing ascending CellChunkMap with multiple chunks in array - private CellChunkMap descMultCCM;// testing descending CellChunkMap with multiple chunks in array + private final boolean smallChunks; private static ChunkCreator chunkCreator; @@ -81,12 +81,13 @@ public class TestCellFlatSet extends TestCase { chunkCreator = ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, globalMemStoreLimit, 0.2f, MemStoreLAB.POOL_INITIAL_SIZE_DEFAULT, null); assertTrue(chunkCreator != null); + smallChunks = false; } else { // chunkCreator with smaller chunk size, so only 3 cell-representations can accommodate a chunk - chunkCreator = ChunkCreator.initialize(64, false, + chunkCreator = ChunkCreator.initialize(SMALL_CHUNK_SIZE, false, globalMemStoreLimit, 0.2f, MemStoreLAB.POOL_INITIAL_SIZE_DEFAULT, null); assertTrue(chunkCreator != null); - + smallChunks = true; } } @@ -125,9 +126,9 @@ public class TestCellFlatSet extends TestCase { ascCCM = setUpCellChunkMap(true); descCCM = setUpCellChunkMap(false); - -// ascMultCCM = setUpCellChunkMap(true); -// descMultCCM = setUpCellChunkMap(false); + if (smallChunks == true) { // check jumbo chunks as well + ascCCM = setUpJumboCellChunkMap(true); + } } /* Create and test ascending CellSet based on CellArrayMap */ @@ -278,8 +279,8 @@ public class TestCellFlatSet extends TestCase { // allocate new chunks and use the data chunk to hold the full data of the cells // and the index chunk to hold the cell-representations - Chunk dataChunk = chunkCreator.getChunk(); - Chunk idxChunk = chunkCreator.getChunk(); + Chunk dataChunk = chunkCreator.getChunk(CompactingMemStore.IndexType.CHUNK_MAP); + Chunk idxChunk = chunkCreator.getChunk(CompactingMemStore.IndexType.CHUNK_MAP); // the array of index chunks to be used as a basis for CellChunkMap Chunk chunkArray[] = new Chunk[8]; // according to test currently written 8 is way enough int chunkArrayIdx = 0; @@ -287,15 +288,16 @@ public class TestCellFlatSet extends TestCase { ByteBuffer idxBuffer = idxChunk.getData(); // the buffers of the chunks ByteBuffer dataBuffer = dataChunk.getData(); - int dataOffset = ChunkCreator.SIZEOF_CHUNK_HEADER; // offset inside data buffer - int idxOffset = ChunkCreator.SIZEOF_CHUNK_HEADER; // skip the space for chunk ID + int dataOffset = ChunkCreator.SIZEOF_CHUNK_HEADER; // offset inside data buffer + int idxOffset = ChunkCreator.SIZEOF_CHUNK_HEADER; // skip the space for chunk ID Cell[] cellArray = asc ? ascCells : descCells; for (Cell kv: cellArray) { // do we have enough space to write the cell data on the data chunk? if (dataOffset + KeyValueUtil.length(kv) > chunkCreator.getChunkSize()) { - dataChunk = chunkCreator.getChunk(); // allocate more data chunks if needed + // allocate more data chunks if needed + dataChunk = chunkCreator.getChunk(CompactingMemStore.IndexType.CHUNK_MAP); dataBuffer = dataChunk.getData(); dataOffset = ChunkCreator.SIZEOF_CHUNK_HEADER; } @@ -304,7 +306,8 @@ public class TestCellFlatSet extends TestCase { // do we have enough space to write the cell-representation on the index chunk? if (idxOffset + ClassSize.CELL_CHUNK_MAP_ENTRY > chunkCreator.getChunkSize()) { - idxChunk = chunkCreator.getChunk(); // allocate more index chunks if needed + // allocate more index chunks if needed + idxChunk = chunkCreator.getChunk(CompactingMemStore.IndexType.CHUNK_MAP); idxBuffer = idxChunk.getData(); idxOffset = ChunkCreator.SIZEOF_CHUNK_HEADER; chunkArray[chunkArrayIdx++] = idxChunk; @@ -317,4 +320,53 @@ public class TestCellFlatSet extends TestCase { return new CellChunkMap(CellComparatorImpl.COMPARATOR,chunkArray,0,NUM_OF_CELLS,!asc); } + + /* Create CellChunkMap with four cells inside the data jumbo chunk. This test is working only + ** with small chunks sized SMALL_CHUNK_SIZE (64) bytes */ + private CellChunkMap setUpJumboCellChunkMap(boolean asc) { + int smallChunkSize = SMALL_CHUNK_SIZE+8; + // allocate new chunks and use the data JUMBO chunk to hold the full data of the cells + // and the normal index chunk to hold the cell-representations + Chunk dataJumboChunk = + chunkCreator.getChunk(CompactingMemStore.IndexType.CHUNK_MAP, smallChunkSize); + Chunk idxChunk = chunkCreator.getChunk(CompactingMemStore.IndexType.CHUNK_MAP); + // the array of index chunks to be used as a basis for CellChunkMap + Chunk[] chunkArray = new Chunk[8]; // according to test currently written 8 is way enough + int chunkArrayIdx = 0; + chunkArray[chunkArrayIdx++] = idxChunk; + + ByteBuffer idxBuffer = idxChunk.getData(); // the buffers of the chunks + ByteBuffer dataBuffer = dataJumboChunk.getData(); + int dataOffset = ChunkCreator.SIZEOF_CHUNK_HEADER; // offset inside data buffer + int idxOffset = ChunkCreator.SIZEOF_CHUNK_HEADER; // skip the space for chunk ID + + Cell[] cellArray = asc ? ascCells : descCells; + + for (Cell kv: cellArray) { + int dataStartOfset = dataOffset; + dataOffset = KeyValueUtil.appendTo(kv, dataBuffer, dataOffset, false); // write deep cell data + + // do we have enough space to write the cell-representation on the index chunk? + if (idxOffset + ClassSize.CELL_CHUNK_MAP_ENTRY > chunkCreator.getChunkSize()) { + // allocate more index chunks if needed + idxChunk = chunkCreator.getChunk(CompactingMemStore.IndexType.CHUNK_MAP); + idxBuffer = idxChunk.getData(); + idxOffset = ChunkCreator.SIZEOF_CHUNK_HEADER; + chunkArray[chunkArrayIdx++] = idxChunk; + } + // write data chunk id + idxOffset = ByteBufferUtils.putInt(idxBuffer, idxOffset, dataJumboChunk.getId()); + idxOffset = ByteBufferUtils.putInt(idxBuffer, idxOffset, dataStartOfset); // offset + idxOffset = ByteBufferUtils.putInt(idxBuffer, idxOffset, KeyValueUtil.length(kv)); // length + idxOffset = ByteBufferUtils.putLong(idxBuffer, idxOffset, kv.getSequenceId()); // seqId + + // Jumbo chunks are working only with one cell per chunk, thus always allocate a new jumbo + // data chunk for next cell + dataJumboChunk = chunkCreator.getChunk(CompactingMemStore.IndexType.CHUNK_MAP,smallChunkSize); + dataBuffer = dataJumboChunk.getData(); + dataOffset = ChunkCreator.SIZEOF_CHUNK_HEADER; + } + + return new CellChunkMap(CellComparatorImpl.COMPARATOR,chunkArray,0,NUM_OF_CELLS,!asc); + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java index 19941b9bcb6..0d5290d7f61 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java @@ -817,7 +817,7 @@ public class TestCompactingMemStore extends TestDefaultMemStore { memstore.clearSnapshot(snapshot.getId()); } - private int addRowsByKeys(final AbstractMemStore hmc, String[] keys) { + protected int addRowsByKeys(final AbstractMemStore hmc, String[] keys) { byte[] fam = Bytes.toBytes("testfamily"); byte[] qf = Bytes.toBytes("testqualifier"); long size = hmc.getActive().keySize(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java index a2e4c475125..37fa85fcbb0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java @@ -98,7 +98,7 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore } // test 1 bucket - long totalCellsLen = addRowsByKeys(memstore, keys1); + long totalCellsLen = addRowsByKeysDataSize(memstore, keys1); long cellBeforeFlushSize = cellBeforeFlushSize(); long cellAfterFlushSize = cellAfterFlushSize(); long totalHeapSize = MutableSegment.DEEP_OVERHEAD + 4 * cellBeforeFlushSize; @@ -144,7 +144,7 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore String[] keys1 = { "A", "A", "B", "C" }; String[] keys2 = { "A", "B", "D" }; - long totalCellsLen1 = addRowsByKeys(memstore, keys1); // INSERT 4 + long totalCellsLen1 = addRowsByKeysDataSize(memstore, keys1); // INSERT 4 long cellBeforeFlushSize = cellBeforeFlushSize(); long cellAfterFlushSize = cellAfterFlushSize(); long totalHeapSize1 = MutableSegment.DEEP_OVERHEAD + 4 * cellBeforeFlushSize; @@ -168,7 +168,7 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore assertEquals(totalCellsLen1, regionServicesForStores.getMemStoreSize()); assertEquals(totalHeapSize1, ((CompactingMemStore) memstore).heapSize()); - long totalCellsLen2 = addRowsByKeys(memstore, keys2); // INSERT 3 (3+3=6) + long totalCellsLen2 = addRowsByKeysDataSize(memstore, keys2); // INSERT 3 (3+3=6) long totalHeapSize2 = 3 * cellBeforeFlushSize; assertEquals(totalCellsLen1 + totalCellsLen2, regionServicesForStores.getMemStoreSize()); assertEquals(totalHeapSize1 + totalHeapSize2, ((CompactingMemStore) memstore).heapSize()); @@ -206,7 +206,7 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore String[] keys2 = { "A", "B", "D" }; String[] keys3 = { "D", "B", "B" }; - long totalCellsLen1 = addRowsByKeys(memstore, keys1); + long totalCellsLen1 = addRowsByKeysDataSize(memstore, keys1); long cellBeforeFlushSize = cellBeforeFlushSize(); long cellAfterFlushSize = cellAfterFlushSize(); long totalHeapSize1 = MutableSegment.DEEP_OVERHEAD + 4 * cellBeforeFlushSize; @@ -227,7 +227,7 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore assertEquals(totalCellsLen1, regionServicesForStores.getMemStoreSize()); assertEquals(totalHeapSize1, ((CompactingMemStore) memstore).heapSize()); - long totalCellsLen2 = addRowsByKeys(memstore, keys2); + long totalCellsLen2 = addRowsByKeysDataSize(memstore, keys2); long totalHeapSize2 = 3 * cellBeforeFlushSize; assertEquals(totalCellsLen1 + totalCellsLen2, regionServicesForStores.getMemStoreSize()); @@ -241,7 +241,7 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore assertEquals(totalCellsLen1 + totalCellsLen2, regionServicesForStores.getMemStoreSize()); assertEquals(totalHeapSize1 + totalHeapSize2, ((CompactingMemStore) memstore).heapSize()); - long totalCellsLen3 = addRowsByKeys(memstore, keys3); + long totalCellsLen3 = addRowsByKeysDataSize(memstore, keys3); long totalHeapSize3 = 3 * cellBeforeFlushSize; assertEquals(totalCellsLen1 + totalCellsLen2 + totalCellsLen3, regionServicesForStores.getMemStoreSize()); @@ -298,7 +298,7 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore memstore.getConfiguration().set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, String.valueOf(compactionType)); ((MyCompactingMemStore)memstore).initiateType(compactionType, memstore.getConfiguration()); - addRowsByKeys(memstore, keys1); + addRowsByKeysDataSize(memstore, keys1); ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline should not compact @@ -307,7 +307,7 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore } assertEquals(0, memstore.getSnapshot().getCellsCount()); - addRowsByKeys(memstore, keys2); // also should only flatten + addRowsByKeysDataSize(memstore, keys2); // also should only flatten int counter2 = 0; for ( Segment s : memstore.getSegments()) { @@ -326,7 +326,7 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore } assertEquals(12, counter3); - addRowsByKeys(memstore, keys3); + addRowsByKeysDataSize(memstore, keys3); int counter4 = 0; for ( Segment s : memstore.getSegments()) { @@ -631,7 +631,7 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore System.currentTimeMillis(), val); // test 1 bucket - long totalCellsLen = addRowsByKeys(memstore, keys1); + int totalCellsLen = addRowsByKeys(memstore, keys1); long oneCellOnCSLMHeapSize = ClassSize.align( ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY + KeyValue.FIXED_OVERHEAD + KeyValueUtil @@ -663,7 +663,8 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore memstore.clearSnapshot(snapshot.getId()); } - private long addRowsByKeys(final AbstractMemStore hmc, String[] keys) { + + private long addRowsByKeysDataSize(final AbstractMemStore hmc, String[] keys) { byte[] fam = Bytes.toBytes("testfamily"); byte[] qf = Bytes.toBytes("testqualifier"); MemStoreSizing memstoreSizing = new MemStoreSizing();