Apache/hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-11120 Update documentation about major compaction algorithm (Misty Stanley-Jones) + addendum fixing CompactionConfiguration

Browse Source
This commit is contained in:
Michael Stack 2014-06-09 20:48:56 -07:00
parent 5851a06412
commit 948b22f082
2 changed files with 35 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionConfiguration.java
View File

@ -46,10 +46,15 @@ public class CompactionConfiguration {
static final Log LOG = LogFactory.getLog(CompactionConfiguration.class); static final Log LOG = LogFactory.getLog(CompactionConfiguration.class);
private static final String CONFIG_PREFIX = "hbase.hstore.compaction."; public static final String HBASE_HSTORE_COMPACTION_RATIO_KEY = "hbase.hstore.compaction.ratio";
public static final String RATIO_KEY = CONFIG_PREFIX + "ratio"; public static final String HBASE_HSTORE_COMPACTION_RATIO_OFFPEAK_KEY =
public static final String MIN_KEY = CONFIG_PREFIX + "min"; "hbase.hstore.compaction.ratio.offpeak";
public static final String MAX_KEY = CONFIG_PREFIX + "max"; public static final String HBASE_HSTORE_COMPACTION_MIN_KEY = "hbase.hstore.compaction.min";
public static final String HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY =
"hbase.hstore.compaction.min.size";
public static final String HBASE_HSTORE_COMPACTION_MAX_KEY = "hbase.hstore.compaction.max";
public static final String HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY =
"hbase.hstore.compaction.max.size";
Configuration conf; Configuration conf;
StoreConfigInformation storeConfigInfo; StoreConfigInformation storeConfigInfo;
@ -68,14 +73,14 @@ public class CompactionConfiguration {
this.conf = conf; this.conf = conf;
this.storeConfigInfo = storeConfigInfo; this.storeConfigInfo = storeConfigInfo;
maxCompactSize = conf.getLong(CONFIG_PREFIX + "max.size", Long.MAX_VALUE); maxCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MAX_SIZE_KEY, Long.MAX_VALUE);
minCompactSize = conf.getLong(CONFIG_PREFIX + "min.size", minCompactSize = conf.getLong(HBASE_HSTORE_COMPACTION_MIN_SIZE_KEY,
storeConfigInfo.getMemstoreFlushSize()); storeConfigInfo.getMemstoreFlushSize());
minFilesToCompact = Math.max(2, conf.getInt(MIN_KEY, minFilesToCompact = Math.max(2, conf.getInt(HBASE_HSTORE_COMPACTION_MIN_KEY,
/*old name*/ conf.getInt("hbase.hstore.compactionThreshold", 3))); /*old name*/ conf.getInt("hbase.hstore.compactionThreshold", 3)));
maxFilesToCompact = conf.getInt(MAX_KEY, 10); maxFilesToCompact = conf.getInt(HBASE_HSTORE_COMPACTION_MAX_KEY, 10);
compactionRatio = conf.getFloat(RATIO_KEY, 1.2F); compactionRatio = conf.getFloat(HBASE_HSTORE_COMPACTION_RATIO_KEY, 1.2F);
offPeekCompactionRatio = conf.getFloat(CONFIG_PREFIX + "ratio.offpeak", 5.0F); offPeekCompactionRatio = conf.getFloat(HBASE_HSTORE_COMPACTION_RATIO_OFFPEAK_KEY, 5.0F);
throttlePoint = conf.getLong("hbase.regionserver.thread.compaction.throttle", throttlePoint = conf.getLong("hbase.regionserver.thread.compaction.throttle",
2 * maxFilesToCompact * storeConfigInfo.getMemstoreFlushSize()); 2 * maxFilesToCompact * storeConfigInfo.getMemstoreFlushSize());

34
src/main/docbkx/book.xml
View File

@ -2645,7 +2645,7 @@ myHtd.setValue(HTableDescriptor.SPLIT_POLICY, MyCustomSplitPolicy.class.getName(
</section> </section>
</section> </section>
<section <section
xml:id="compaction"> xml:id="compaction">
<title>Compaction</title> <title>Compaction</title>
<para><firstterm>Compaction</firstterm> is an operation which reduces the number of <para><firstterm>Compaction</firstterm> is an operation which reduces the number of
@ -2712,9 +2712,9 @@ myHtd.setValue(HTableDescriptor.SPLIT_POLICY, MyCustomSplitPolicy.class.getName(
table which shows their descriptions, defaults, and implications of changing table which shows their descriptions, defaults, and implications of changing
them.</para> them.</para>
<formalpara> <formalpara xml:id="exploringcompaction.policy">
<title>The<link <title>The<link
xlink:href="https://issues.apache.org/jira/browse/HBASE-7842">Exploring Compaction Policy</link></title> xlink:href="https://issues.apache.org/jira/browse/HBASE-7842">ExploringCompaction Policy</link></title>
<para><link <para><link
xlink:href="https://issues.apache.org/jira/browse/HBASE-7842">HBASE-7842</link> xlink:href="https://issues.apache.org/jira/browse/HBASE-7842">HBASE-7842</link>
was introduced in HBase 0.96 and represents a major change in the algorithms for was introduced in HBase 0.96 and represents a major change in the algorithms for
@ -2751,16 +2751,18 @@ myHtd.setValue(HTableDescriptor.SPLIT_POLICY, MyCustomSplitPolicy.class.getName(
<varname>hbase.hregion.memstore.flush.size</varname>. Normally, minor <varname>hbase.hregion.memstore.flush.size</varname>. Normally, minor
compactions will alleviate this situation. However, if the normal compaction compactions will alleviate this situation. However, if the normal compaction
algorithm do not find any normally-eligible StoreFiles, a major compaction is algorithm do not find any normally-eligible StoreFiles, a major compaction is
the only way to get out of this situation, and is forced. This is also called a the only way to get out of this situation, and is forced.</para>
size-based or size-triggered major compaction.</para> <para>If you are using the ExploringCompaction policy, the set of files to
compact is always selected, and will not trigger a major compaction. See <xref
linkend="exploringcompaction.policy" />.</para>
</listitem> </listitem>
</varlistentry> </varlistentry>
<varlistentry> <varlistentry>
<term>If this compaction was user-requested, do a major compaction.</term> <term>If this compaction was user-requested, perform the requested type of compaction.</term>
<listitem> <listitem>
<para>Compactions can run on a schedule or can be initiated manually. If a <para>Compactions can run on a schedule or can be initiated manually. If a
compaction is requested manually, it is always a major compaction. If the compaction is requested manually, HBase always runs that type of compaction. If the
compaction is user-requested, the major compaction still happens even if the are user requests a major compaction, the major compaction still runs even if the are
more than <varname>hbase.hstore.compaction.max</varname> files that need more than <varname>hbase.hstore.compaction.max</varname> files that need
compaction.</para> compaction.</para>
</listitem> </listitem>
@ -2841,9 +2843,9 @@ myHtd.setValue(HTableDescriptor.SPLIT_POLICY, MyCustomSplitPolicy.class.getName(
<textobject> <textobject>
<para>This table contains the main configuration parameters for compaction. This <para>This table contains the main configuration parameters for compaction. This
list is not exhaustive. To tune these parameters from the defaults, edit the list is not exhaustive. To tune these parameters from the defaults, edit the
<filename>hbase-default.xml</filename> file. For a full list of all <filename>hbase-default.xml</filename> file. For a full list of all
configuration parameters available, see <xref configuration parameters available, see <xref
linkend="config.files" />.</para> linkend="config.files" /></para>
</textobject> </textobject>
<tgroup <tgroup
cols="3"> cols="3">
@ -2857,8 +2859,12 @@ myHtd.setValue(HTableDescriptor.SPLIT_POLICY, MyCustomSplitPolicy.class.getName(
<tbody> <tbody>
<row> <row>
<entry>hbase.hstore.compaction.min</entry> <entry>hbase.hstore.compaction.min</entry>
<entry>The minimum number of files which must be eligible for compaction before <entry><para>The minimum number of files which must be eligible for compaction
compaction can run.</entry> before compaction can run.</para>
<para>In previous versions, the parameter
<code>hbase.hstore.compaction.min</code> was called
<code>hbase.hstore.compactionThreshold</code>.</para>
</entry>
<entry>3</entry> <entry>3</entry>
</row> </row>
<row> <row>
@ -2871,13 +2877,13 @@ myHtd.setValue(HTableDescriptor.SPLIT_POLICY, MyCustomSplitPolicy.class.getName(
<entry>hbase.hstore.compaction.min.size</entry> <entry>hbase.hstore.compaction.min.size</entry>
<entry>A StoreFile smaller than this size (in bytes) will always be eligible for <entry>A StoreFile smaller than this size (in bytes) will always be eligible for
minor compaction.</entry> minor compaction.</entry>
<entry>10</entry> <entry>128 MB</entry>
</row> </row>
<row> <row>
<entry>hbase.hstore.compaction.max.size</entry> <entry>hbase.hstore.compaction.max.size</entry>
<entry>A StoreFile larger than this size (in bytes) will be excluded from minor <entry>A StoreFile larger than this size (in bytes) will be excluded from minor
compaction.</entry> compaction.</entry>
<entry>1000</entry> <entry>Long.MAX_VALUE</entry>
</row> </row>
<row> <row>
<entry>hbase.store.compaction.ratio</entry> <entry>hbase.store.compaction.ratio</entry>