hbase/conf/hbase-default.xml

282 lines
10 KiB
XML
Raw Normal View History

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
/**
* Copyright 2007 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-->
<configuration>
<property>
<name>hbase.master</name>
<value>local</value>
<description>The host and port that the HBase master runs at.
A value of 'local' runs the master and a regionserver in
a single process.
</description>
</property>
<property>
<name>hbase.rootdir</name>
<value>file:///tmp/hbase-${user.home}/hbase</value>
<description>The directory shared by region servers.
Should be fully-qualified to include the filesystem to use.
E.g: hdfs://NAMENODE_SERVER:PORT/HBASE_ROOTDIR
</description>
</property>
<property>
<name>hbase.master.info.port</name>
<value>60010</value>
<description>The port for the hbase master web UI
Set to -1 if you do not want the info server to run.
</description>
</property>
<property>
<name>hbase.master.info.bindAddress</name>
<value>0.0.0.0</value>
<description>The address for the hbase master web UI
</description>
</property>
<property>
<name>hbase.regionserver</name>
<value>0.0.0.0:60020</value>
<description>The host and port a HBase region server runs at.
</description>
</property>
<property>
<name>hbase.regionserver.info.port</name>
<value>60030</value>
<description>The port for the hbase regionserver web UI
Set to -1 if you do not want the info server to run.
</description>
</property>
<property>
<name>hbase.regionserver.info.bindAddress</name>
<value>0.0.0.0</value>
<description>The address for the hbase regionserver web UI
</description>
</property>
<property>
<name>hbase.regionserver.class</name>
<value>org.apache.hadoop.hbase.ipc.HRegionInterface</value>
<description>An interface that is assignable to HRegionInterface. Used in HClient for
opening proxy to remote region server.
</description>
</property>
<property>
<name>hbase.client.pause</name>
<value>10000</value>
<description>General client pause value. Used mostly as value to wait
before running a retry of a failed get, region lookup, etc.</description>
</property>
<property>
<name>hbase.client.retries.number</name>
<value>5</value>
<description>Maximum retries. Used as maximum for all retryable
operations such as fetching of the root region from root region
server, getting a cell's value, starting a row update, etc.
Default: 5.
</description>
</property>
<property>
<name>hbase.master.meta.thread.rescanfrequency</name>
<value>60000</value>
<description>How long the HMaster sleeps (in milliseconds) between scans of
the root and meta tables.
</description>
</property>
<property>
<name>hbase.master.lease.period</name>
<value>60000</value>
<description>HMaster server lease period in milliseconds. Default is
60 seconds. Region servers must report in within this period else
HADOOP-1523 'Hung region servers waiting on write locks' On shutdown, region servers and masters were just cancelling leases without letting 'lease expired' code run -- code to clean up outstanding locks in region server. Outstanding read locks were getting in the way of region server getting necessary write locks needed for the shutdown process. Also, cleaned up messaging around shutdown so its clean -- no timeout messages as region servers try to talk to a master that has already shutdown -- even when region servers take their time going down. M src/contrib/hbase/conf/hbase-default.xml Make region server timeout 30 seconds instead of 3 minutes. Clients retry anyways. Make so its likely region servers report in their shutdown message before their lease expires on master. M src/contrib/hbase/src/java/org/apache/hadoop/hbase/Leases.java (closeAfterLeasesExpire): Added. * src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionServer.java Added comments. (stop): Converted from public to default access (master shuts down regionservers). (run): Use leases.closeAfterLeasesExpire instead of leases.close. Changed log of main thread exit from DEBUG to INFO. * src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java (letRegionsServersShutdown): Add better explaination of shutdown process to method doc. Changed timeout waits from hbase.regionserver.msginterval to threadWakeFrequency. (regionServerReport): If closing, we used to immediately respond to region server with a MSG_REGIONSERVER_STOP. This meant that we avoided handling of the region servers MSG_REPORT_EXITING sent on shutdown so region servers had no chance to cancel their lease in the master. Reordered. Moved sending of MSG_REGIONSERVER_STOP to after handling of MSG_REPORT_EXITING. Also, in handling of MSG_REGIONSERER_STOP removed cancelling of leases. Let leases expire normally (or get cancelled when the region server comes in with MSG_RPORT_EXITING). * src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMsg.java (MSG_REGIONSERVER_STOP_IN_ARRAY): Added. git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk/src/contrib/hbase@552376 13f79535-47bb-0310-9956-ffa450edef68
2007-07-01 20:47:13 -04:00
they are considered dead. On loaded cluster, may need to up this
period.</description>
</property>
HBASE-501 Empty region server address in info:server entry and a startcode of -1 in .META. M conf/hbase-default.xml Add hbase.hbasemaster.maxregionopen property. M src/java/org/apache/hadoop/hbase/HStore.java Change way we log. Do way less. Just emit sums of edits applied and skipped rather than individual edits. M src/java/org/apache/hadoop/hbase/HRegionServer.java Make sleeper instance a local rather than data member. (reportForDuty): Take a sleeper instance. (run): Removed redundant wrap of a 'for' by a 'while'. (constructor): If IOE, do not offline the region. Seen to be an overreaction. M src/java/org/apache/hadoop/hbase/HLog.java Don't output map of all files being cleaned everytime a new entry is added; instead just log new entry. Remove emission of every 10k edits. M src/java/org/apache/hadoop/hbase/HMaster.java Up default for maxregionopen. Was seeing that playing edits could take a long time (mostly because we used log every edit) but no harm in this being longer. On REPORT_CLOSE, emit region info, not just region so can see the properties (W/o, made it hard to figure who was responsible for offlining). Add logging of attempt # in shutdown processing. Add logging of state flags passed to the close region. Helps debugging. Also in close offline ONLY if we are NOT reassigning the region (jimk find). M src/java/org/apache/hadoop/hbase/util/Sleeper.java Add logging of extraordinary sleeps or calculated periods (suspicion is that we're sleeping way longer on loaded machies and the regionserver appears hung). git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@636849 13f79535-47bb-0310-9956-ffa450edef68
2008-03-13 15:37:21 -04:00
<property>
<name>hbase.hbasemaster.maxregionopen</name>
<value>60000</value>
<description>Period to wait for a region open. If regionserver
takes longer than this interval, assign to a new regionserver.
</description>
</property>
<property>
<name>hbase.regionserver.lease.period</name>
HADOOP-1523 'Hung region servers waiting on write locks' On shutdown, region servers and masters were just cancelling leases without letting 'lease expired' code run -- code to clean up outstanding locks in region server. Outstanding read locks were getting in the way of region server getting necessary write locks needed for the shutdown process. Also, cleaned up messaging around shutdown so its clean -- no timeout messages as region servers try to talk to a master that has already shutdown -- even when region servers take their time going down. M src/contrib/hbase/conf/hbase-default.xml Make region server timeout 30 seconds instead of 3 minutes. Clients retry anyways. Make so its likely region servers report in their shutdown message before their lease expires on master. M src/contrib/hbase/src/java/org/apache/hadoop/hbase/Leases.java (closeAfterLeasesExpire): Added. * src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionServer.java Added comments. (stop): Converted from public to default access (master shuts down regionservers). (run): Use leases.closeAfterLeasesExpire instead of leases.close. Changed log of main thread exit from DEBUG to INFO. * src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java (letRegionsServersShutdown): Add better explaination of shutdown process to method doc. Changed timeout waits from hbase.regionserver.msginterval to threadWakeFrequency. (regionServerReport): If closing, we used to immediately respond to region server with a MSG_REGIONSERVER_STOP. This meant that we avoided handling of the region servers MSG_REPORT_EXITING sent on shutdown so region servers had no chance to cancel their lease in the master. Reordered. Moved sending of MSG_REGIONSERVER_STOP to after handling of MSG_REPORT_EXITING. Also, in handling of MSG_REGIONSERER_STOP removed cancelling of leases. Let leases expire normally (or get cancelled when the region server comes in with MSG_RPORT_EXITING). * src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMsg.java (MSG_REGIONSERVER_STOP_IN_ARRAY): Added. git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk/src/contrib/hbase@552376 13f79535-47bb-0310-9956-ffa450edef68
2007-07-01 20:47:13 -04:00
<value>30000</value>
<description>HRegion server lease period in milliseconds. Default is
HADOOP-1523 'Hung region servers waiting on write locks' On shutdown, region servers and masters were just cancelling leases without letting 'lease expired' code run -- code to clean up outstanding locks in region server. Outstanding read locks were getting in the way of region server getting necessary write locks needed for the shutdown process. Also, cleaned up messaging around shutdown so its clean -- no timeout messages as region servers try to talk to a master that has already shutdown -- even when region servers take their time going down. M src/contrib/hbase/conf/hbase-default.xml Make region server timeout 30 seconds instead of 3 minutes. Clients retry anyways. Make so its likely region servers report in their shutdown message before their lease expires on master. M src/contrib/hbase/src/java/org/apache/hadoop/hbase/Leases.java (closeAfterLeasesExpire): Added. * src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionServer.java Added comments. (stop): Converted from public to default access (master shuts down regionservers). (run): Use leases.closeAfterLeasesExpire instead of leases.close. Changed log of main thread exit from DEBUG to INFO. * src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java (letRegionsServersShutdown): Add better explaination of shutdown process to method doc. Changed timeout waits from hbase.regionserver.msginterval to threadWakeFrequency. (regionServerReport): If closing, we used to immediately respond to region server with a MSG_REGIONSERVER_STOP. This meant that we avoided handling of the region servers MSG_REPORT_EXITING sent on shutdown so region servers had no chance to cancel their lease in the master. Reordered. Moved sending of MSG_REGIONSERVER_STOP to after handling of MSG_REPORT_EXITING. Also, in handling of MSG_REGIONSERER_STOP removed cancelling of leases. Let leases expire normally (or get cancelled when the region server comes in with MSG_RPORT_EXITING). * src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMsg.java (MSG_REGIONSERVER_STOP_IN_ARRAY): Added. git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk/src/contrib/hbase@552376 13f79535-47bb-0310-9956-ffa450edef68
2007-07-01 20:47:13 -04:00
30 seconds. Clients must report in within this period else they are
considered dead.</description>
</property>
<property>
<name>hbase.server.thread.wakefrequency</name>
<value>10000</value>
<description>Time to sleep in between searches for work (in milliseconds).
Used as sleep interval by service threads such as META scanner and log roller.
</description>
</property>
<property>
<name>hbase.regionserver.handler.count</name>
<value>10</value>
<description>Count of RPC Server instances spun up on RegionServers
Same property is used by the HMaster for count of master handlers.
Default is 10.
</description>
</property>
<property>
<name>hbase.regionserver.msginterval</name>
<value>3000</value>
<description>Interval between messages from the RegionServer to HMaster
in milliseconds. Default is 15. Set this value low if you want unit
tests to be responsive.
</description>
</property>
<property>
<name>hbase.regionserver.maxlogentries</name>
<value>30000</value>
<description>Rotate the HRegion HLogs when count of entries exceeds this
value. Default: 30,000. Value is checked by a thread that runs every
hbase.server.thread.wakefrequency.
</description>
</property>
<property>
<name>hbase.regionserver.optionalcacheflushinterval</name>
<value>1800000</value>
<description>
Amount of time to wait since the last time a region was flushed before
invoking an optional cache flush (An optional cache flush is a
flush even though memcache is not at the memcache.flush.size).
Default: 30 minutes (in miliseconds)
</description>
</property>
<property>
<name>hbase.regionserver.optionallogrollinterval</name>
<value>1800000</value>
<description>
Amount of time to wait since the last time a the region server's log was
rolled before invoking an optional log roll (An optional log roll is a
one in which the log does not contain hbase.regionserver.maxlogentries).
Default: 30 minutes (in miliseconds)
</description>
</property>
<property>
<name>hbase.hregion.memcache.flush.size</name>
<value>67108864</value>
<description>
A HRegion memcache will be flushed to disk if size of the memcache
exceeds this number of bytes. Value is checked by a thread that runs
every hbase.server.thread.wakefrequency.
</description>
</property>
<property>
<name>hbase.hregion.memcache.block.multiplier</name>
<value>1</value>
<description>
Block updates if memcache has hbase.hregion.block.memcache
time hbase.hregion.flush.size bytes. Useful preventing
runaway memcache during spikes in update traffic. Without an
upper-bound, memcache fills such that when it flushes the
resultant flush files take a long time to compact or split, or
worse, we OOME.
</description>
</property>
<property>
<name>hbase.hregion.max.filesize</name>
<value>268435456</value>
<description>
Maximum HStoreFile size. If any one of a column families' HStoreFiles has
grown to exceed this value, the hosting HRegion is split in two.
Default: 256M.
</description>
</property>
<property>
HADOOP-1644 [hbase] Compactions should not block updates Disentangles flushes and compactions; flushes can proceed while a compaction is happening. Also, don't compact unless we hit compaction threshold: i.e. don't automatically compact on HRegion startup so regions can come online the faster. M src/contrib/hbase/conf/hbase-default.xml (hbase.hregion.compactionThreashold): Moved to be a hstore property as part of encapsulating compaction decision inside hstore. M src/contrib/hbase/src/test/org/apache/hadoop/hbase/HBaseTestCase.java Refactored. Moved here generalized content loading code that can be shared by tests. Add to setup and teardown the setup and removal of local test dir (if it exists). M src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestCompare.java Added test of HStoreKey compare (It works other than one would at first expect). M src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestSplit.java Bulk of content loading code has been moved up into the parent class. M src/contrib/hbase/src/java/org/apache/hadoop/hbase/HConnectionManager.java (tableExists): Restore to a check of if the asked-for table is in list of tables. As it was, a check for tableExists would just wait on all timeouts and retries to expire and then report table does not exist.. Fixed up debug message listing regions of a table. Added protection against meta table not having a COL_REGINFO (Seen in cluster testing -- probably a bug in row removal). M src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStoreFile.java Loading store files, even if it was noticed that there was no corresponding map file, was still counting file as valid. Also fix merger -- was constructing MapFile.Reader directly rather than asking HStoreFile for the reader (HStoreFile knows how to do MapFile references) (rename): Added check that move succeeded and logging. In cluster-testing, the hdfs move of compacted file into place has failed on occasion (Need more info). M src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStore.java Encapsulate ruling on whether a compaction should take place inside HStore. Added reading of the compactionThreshold her. Compaction threshold is currently just number of store files. Later may include other factors such as count of reference files. Cleaned up debug messages around reconstruction log. Removed compaction if size > 1 from constructor. Let compaction happen after we've been deployed (Compactions that happen while we are online can continue to take updates. Compaction in the constructor puts off our being able to take in updates). (close): Changed so it now returns set of store files. This used to be done by calls to flush. Since flush and compaction have been disentangled, a compaction can come in after flush and the list of files could be off. Having it done by close, can be sure list of files is complete. (flushCache): No longer returns set of store files. Added 'merging compaction' where we pick an arbitrary store file from disk and merge into it the content of memcache (Needs work). (getAllMapFiles): Renamed getAllStoreFiles. (needsCompaction): Added. (compactHelper): Added passing of maximum sequence number if already calculated. If compacting one file only, we used skip without rewriting the info file. Fixed. Refactored. Moved guts to new compact(outFile, listOfStores) method. (compact, CompactionReader): Added overrides and interface to support 'merging compaction' that takes files and memcache. In compaction, if we failed the move of the compacted file, all data had already been deleted. Changing, so deletion happens after confirmed move of compacted file. (getFull): Fixed bug where NPE when read of maps came back null. Revealed by our NOT compacting stores on startup. Meant could be two backing stores one of which had no data regards queried key. (getNMaps): Renamed countOfStoreFiles. (toString): Added. M src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStoreKey.java Added comment on 'odd'-looking comparison. M src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionServer.java Javadoc edit. M src/contrib/hbase/src/java/org/apache/hadoop/hbase/HLogEdit.java Only return first 128 bytes of value when toStringing (On cluster, was returning complete web pages in log). M src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java Removed confusing debug message (made sense once -- but not now). Test rootRegionLocation for null before using it (can be null). M src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMemcache.java Added comment that delete behavior needs study. M src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegion.java Fixed merge so it doesn't do the incremental based off files returned by flush. Instead all is done in the one go after region closes (using files returned by close). Moved duplicated code to new filesByFamily method. (WriteState): Removed writesOngoing in favor of compacting and flushing flags. (flushCache): No longer returns list of files. M src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/Writables.java Fix javadoc. git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk/src/contrib/hbase@566459 13f79535-47bb-0310-9956-ffa450edef68
2007-08-15 21:07:51 -04:00
<name>hbase.hstore.compactionThreshold</name>
<value>3</value>
<description>
If more than this number of HStoreFiles in any one HStore
(one HStoreFile is written per flush of memcache) then a compaction
is run to rewrite all HStoreFiles files as one. Larger numbers
put off compaction but when it runs, it takes longer to complete.
During a compaction, updates cannot be flushed to disk. Long
compactions require memory sufficient to carry the logging of
all updates across the duration of the compaction.
If too large, clients timeout during compaction.
</description>
</property>
<property>
<name>hbase.regionserver.thread.splitcompactcheckfrequency</name>
<value>20000</value>
<description>How often a region server runs the split/compaction check.
</description>
</property>
<property>
<name>hbase.io.index.interval</name>
<value>32</value>
<description>The interval at which we record offsets in hbase
store files/mapfiles. Default for stock mapfiles is 128. Index
files are read into memory. If there are many of them, could prove
a burden. If so play with the hadoop io.map.index.skip property and
skip every nth index member when reading back the index into memory.
</description>
</property>
<property>
<name>hbase.io.seqfile.compression.type</name>
<value>NONE</value>
<description>The compression type for hbase sequencefile.Writers
such as hlog.
</description>
</property>
<property>
<name>hbase.hstore.blockCache.blockSize</name>
<value>65536</value>
<description>The size of each block in any block caches.
</description>
</property>
HADOOP-1375 a simple parser for hbase M src/contrib/hbase/NOTICE.txt Add notice of udanax contributions. Msrc/contrib/hbase/conf/hbase-default.xml (hbaseshell.jline.bell.enabled): Added. M src/contrib/hbase/CHANGES.txt (hadoop-1375) Added. M src/contrib/hbase/src/java/org/apache/hadoop/hbase/package.html Add note on how to start up hbase shell M src/contrib/hbase/bin/hbase Add 'shell'. Remove 'client' (shell does what it used do and more). Removed all reader and logreader until better developed. Starting up a reader or logreader on a running hbase system could do damage). M src/contrib/hbase/build.xml Add a javacc target to generate content of shell/generated subpackage. A src/contrib/hbase/src/test/org/apache/hadoop/hbase/shell/TestHBaseShell.java A src/contrib/hbase/src/java/org/apache/hadoop/hbase/Shell.java A src/contrib/hbase/src/java/org/apache/hadoop/hbase/shell/DeleteCommand.java A src/contrib/hbase/src/java/org/apache/hadoop/hbase/shell/CreateCommand.java A src/contrib/hbase/src/java/org/apache/hadoop/hbase/shell/DropCommand.java A src/contrib/hbase/src/java/org/apache/hadoop/hbase/shell/InsertCommand.java A src/contrib/hbase/src/java/org/apache/hadoop/hbase/shell/CommandFactory.java A src/contrib/hbase/src/java/org/apache/hadoop/hbase/shell/HelpContents.java A src/contrib/hbase/src/java/org/apache/hadoop/hbase/shell/ExitCommand.java A src/contrib/hbase/src/java/org/apache/hadoop/hbase/shell/ConsoleTable.java A src/contrib/hbase/src/java/org/apache/hadoop/hbase/shell/DescCommand.java A src/contrib/hbase/src/java/org/apache/hadoop/hbase/shell/SelectCommand.java A src/contrib/hbase/src/java/org/apache/hadoop/hbase/shell/Command.java A src/contrib/hbase/src/java/org/apache/hadoop/hbase/shell/ShowCommand.java A src/contrib/hbase/src/java/org/apache/hadoop/hbase/shell/BasicCommand.java A src/contrib/hbase/src/java/org/apache/hadoop/hbase/shell/HelpManager.java A src/contrib/hbase/src/java/org/apache/hadoop/hbase/shell/ReturnMsg.java A src/contrib/hbase/src/java/org/apache/hadoop/hbase/shell/HelpCommand.java A src/contrib/hbase/src/java/org/apache/hadoop/hbase/shell/HBaseShell.jj Added. A src/contrib/hbase/src/java/org/apache/hadoop/hbase/shell/generated/Token.java A src/contrib/hbase/src/java/org/apache/hadoop/hbase/shell/generated/TokenMgrError.java A src/contrib/hbase/src/java/org/apache/hadoop/hbase/shell/generated/SimpleCharStream.java A src/contrib/hbase/src/java/org/apache/hadoop/hbase/shell/generated/ParserTokenManager.java A src/contrib/hbase/src/java/org/apache/hadoop/hbase/shell/generated/ParseException.java A src/contrib/hbase/src/java/org/apache/hadoop/hbase/shell/generated/ParserConstants.java A src/contrib/hbase/src/java/org/apache/hadoop/hbase/shell/generated/Parser.java Added javacc generated files. git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk/src/contrib/hbase@555415 13f79535-47bb-0310-9956-ffa450edef68
2007-07-11 17:54:15 -04:00
<!-- HbaseShell Configurations -->
<property>
<name>hbaseshell.jline.bell.enabled</name>
<value>true</value>
<description>
if true, enable audible keyboard bells if an alert is required.
</description>
</property>
<property>
<name>hbaseshell.formatter</name>
<value>org.apache.hadoop.hbase.hql.formatter.AsciiTableFormatter</value>
<description>TableFormatter to use outputting HQL result sets.
</description>
</property>
<property>
<name>hbase.regionserver.globalMemcacheLimit</name>
<value>536870912</value>
<description>Maximum size of all memcaches in a region server before new
updates are blocked and flushes are forced. Defaults to 512MB.
</description>
</property>
<property>
<name>hbase.regionserver.globalMemcacheLimitlowMark</name>
<value>256435456</value>
<description>When memcaches are being forced to flush to make room in
memory, keep flushing until we hit this mark. Defaults to 256MB. Setting
this value equal to hbase.regionserver.globalmemcachelimit causes the
minimum possible flushing to occur when updates are blocked due to
memcache limiting.
</description>
</property>
</configuration>