diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java index fc3e5ba6692..3886e7d9282 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java @@ -43,12 +43,12 @@ import org.apache.hadoop.hbase.wal.WAL; /** * A memstore implementation which supports in-memory compaction. * A compaction pipeline is added between the active set and the snapshot data structures; - * it consists of a list of kv-sets that are subject to compaction. - * Like the snapshot, all pipeline components are read-only; updates only affect the active set. + * it consists of a list of segments that are subject to compaction. + * Like the snapshot, all pipeline segments are read-only; updates only affect the active set. * To ensure this property we take advantage of the existing blocking mechanism -- the active set * is pushed to the pipeline while holding the region's updatesLock in exclusive mode. - * Periodically, a compaction is applied in the background to all pipeline components resulting - * in a single read-only component. The ``old'' components are discarded when no scanner is reading + * Periodically, a compaction is applied in the background to all pipeline segments resulting + * in a single read-only component. The ``old'' segments are discarded when no scanner is reading * them. */ @InterfaceAudience.Private @@ -62,7 +62,7 @@ public class CompactingMemStore extends AbstractMemStore { // Default fraction of in-memory-flush size w.r.t. flush-to-disk size public static final String IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY = "hbase.memstore.inmemoryflush.threshold.factor"; - private static final double IN_MEMORY_FLUSH_THRESHOLD_FACTOR_DEFAULT = 0.1; + private static final double IN_MEMORY_FLUSH_THRESHOLD_FACTOR_DEFAULT = 0.014; private static final Logger LOG = LoggerFactory.getLogger(CompactingMemStore.class); private HStore store; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactionStrategy.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactionStrategy.java index 2f16055fbce..6d28e373521 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactionStrategy.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactionStrategy.java @@ -42,7 +42,7 @@ public abstract class MemStoreCompactionStrategy { // The upper bound for the number of segments we store in the pipeline prior to merging. public static final String COMPACTING_MEMSTORE_THRESHOLD_KEY = "hbase.hregion.compacting.pipeline.segments.limit"; - public static final int COMPACTING_MEMSTORE_THRESHOLD_DEFAULT = 4; + public static final int COMPACTING_MEMSTORE_THRESHOLD_DEFAULT = 2; /** * Types of actions to be done on the pipeline upon MemStoreCompaction invocation. diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java index bea8c4fed6d..70d6970bdad 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java @@ -848,8 +848,10 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore // set memstore to flat into CellChunkMap MemoryCompactionPolicy compactionType = MemoryCompactionPolicy.BASIC; + memstore.getConfiguration().setInt(MemStoreCompactionStrategy + .COMPACTING_MEMSTORE_THRESHOLD_KEY, 4); memstore.getConfiguration().set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, - String.valueOf(compactionType)); + String.valueOf(compactionType)); ((MyCompactingMemStore) memstore).initiateType(compactionType, memstore.getConfiguration()); ((CompactingMemStore) memstore).setIndexType(CompactingMemStore.IndexType.CHUNK_MAP); diff --git a/src/main/asciidoc/_chapters/inmemory_compaction.adoc b/src/main/asciidoc/_chapters/inmemory_compaction.adoc index da3abb13278..21513f395f2 100644 --- a/src/main/asciidoc/_chapters/inmemory_compaction.adoc +++ b/src/main/asciidoc/_chapters/inmemory_compaction.adoc @@ -52,7 +52,7 @@ In this section we describe how to enable Accordion and the available configurat To enable in-memory compactions, set the _IN_MEMORY_COMPACTION_ attribute on per column family where you want the behavior. The _IN_MEMORY_COMPACTION_ -attribute can have one of three values. +attribute can have one of four values. * _NONE_: No in-memory compaction. * _BASIC_: Basic policy enables flushing and keeps a pipeline of flushes until we trip the pipeline maximum threshold and then we flush to disk. No in-memory compaction but can help throughput as data is moved from the profligate, native ConcurrentSkipListMap data-type to more compact (and efficient) data types. @@ -94,11 +94,12 @@ to set the in-memory attribute to _NONE_). When an in-memory flush happens is calculated by dividing the configured region flush size (Set in the table descriptor or read from _hbase.hregion.memstore.flush.size_) by the number of column families and then multiplying by -_hbase.memstore.inmemoryflush.threshold.factor_ (default is 0.1). +_hbase.memstore.inmemoryflush.threshold.factor_ (default is 0.014 - a little less than the size +of a chunk). The number of flushes carried by the pipeline is monitored so as to fit within the bounds of memstore sizing but you can also set a maximum on the number of flushes total by setting -_hbase.hregion.compacting.pipeline.segments.limit_. Default is 4. +_hbase.hregion.compacting.pipeline.segments.limit_. Default is 2. When a column family Store is created, it says what memstore type is in effect. As of this writing there is the old-school _DefaultMemStore_ which fills a _ConcurrentSkipListMap_ and then flushes