diff --git a/bin/hbase b/bin/hbase index 8678eb8fcc9..7e7bf0d3464 100755 --- a/bin/hbase +++ b/bin/hbase @@ -114,18 +114,23 @@ if [ -f "$HBASE_HOME/conf/hbase-env-$COMMAND.sh" ]; then . "$HBASE_HOME/conf/hbase-env-$COMMAND.sh" fi -# check envvars which might override default args -if [ "$HBASE_HEAPSIZE" != "" ]; then - SUFFIX="m" - if [ "${HBASE_HEAPSIZE: -1}" == "m" ] || [ "${HBASE_HEAPSIZE: -1}" == "M" ]; then - SUFFIX="" - fi - if [ "${HBASE_HEAPSIZE: -1}" == "g" ] || [ "${HBASE_HEAPSIZE: -1}" == "G" ]; then - SUFFIX="" - fi - #echo "run with heapsize $HBASE_HEAPSIZE" - JAVA_HEAP_MAX="-Xmx""$HBASE_HEAPSIZE""$SUFFIX" - #echo $JAVA_HEAP_MAX +add_size_suffix() { + # add an 'm' suffix if the argument is missing one, otherwise use whats there + local val="$1" + local lastchar=${val: -1} + if [[ "mMgG" == *$lastchar* ]]; then + echo $val + else + echo ${val}m + fi +} + +if [[ -n "$HBASE_HEAPSIZE" ]]; then + JAVA_HEAP_MAX="-Xmx$(add_size_suffix $HBASE_HEAPSIZE)" +fi + +if [[ -n "$HBASE_OFFHEAPSIZE" ]]; then + JAVA_OFFHEAP_MAX="-XX:MaxDirectMemorySize=$(add_size_suffix $HBASE_OFFHEAPSIZE)" fi # so that filenames w/ spaces are handled correctly in loops below @@ -380,10 +385,11 @@ else HBASE_OPTS="$HBASE_OPTS -Dhbase.security.logger=${HBASE_SECURITY_LOGGER:-INFO,NullAppender}" fi +HEAP_SETTINGS="$JAVA_HEAP_MAX $JAVA_OFFHEAP_MAX" # Exec unless HBASE_NOEXEC is set. export CLASSPATH if [ "${HBASE_NOEXEC}" != "" ]; then - "$JAVA" -Dproc_$COMMAND -XX:OnOutOfMemoryError="kill -9 %p" $JAVA_HEAP_MAX $HBASE_OPTS $CLASS "$@" + "$JAVA" -Dproc_$COMMAND -XX:OnOutOfMemoryError="kill -9 %p" $HEAP_SETTINGS $HBASE_OPTS $CLASS "$@" else - exec "$JAVA" -Dproc_$COMMAND -XX:OnOutOfMemoryError="kill -9 %p" $JAVA_HEAP_MAX $HBASE_OPTS $CLASS "$@" + exec "$JAVA" -Dproc_$COMMAND -XX:OnOutOfMemoryError="kill -9 %p" $HEAP_SETTINGS $HBASE_OPTS $CLASS "$@" fi diff --git a/bin/hbase.cmd b/bin/hbase.cmd index 42eb7b50299..e30e1cdbb35 100644 --- a/bin/hbase.cmd +++ b/bin/hbase.cmd @@ -88,12 +88,17 @@ if "%hbase-command%"=="" ( ) set JAVA_HEAP_MAX=-Xmx1000m +set JAVA_OFFHEAP_MAX="" rem check envvars which might override default args if defined HBASE_HEAPSIZE ( set JAVA_HEAP_MAX=-Xmx%HBASE_HEAPSIZE%m ) +if defined HBASE_OFFHEAPSIZE ( + set JAVA_OFFHEAP_MAX=-XX:MaxDirectMemory=%HBASE_OFFHEAPSIZE%m +) + set CLASSPATH=%HBASE_CONF_DIR%;%JAVA_HOME%\lib\tools.jar rem Add maven target directory @@ -288,7 +293,8 @@ if not defined HBASE_SECURITY_LOGGER ( ) set HBASE_OPTS=%HBASE_OPTS% -Dhbase.security.logger="%HBASE_SECURITY_LOGGER%" -set java_arguments=%JAVA_HEAP_MAX% %HBASE_OPTS% -classpath "%CLASSPATH%" %CLASS% %hbase-command-arguments% +set HEAP_SETTINGS="%JAVA_HEAP_MAX% %JAVA_OFFHEAP_MAX%" +set java_arguments=%HEAP_SETTINGS% %HBASE_OPTS% -classpath "%CLASSPATH%" %CLASS% %hbase-command-arguments% if defined service_entry ( call :makeServiceXml %java_arguments% diff --git a/conf/hbase-env.cmd b/conf/hbase-env.cmd index 26570dbfb28..bf7f25aa97d 100644 --- a/conf/hbase-env.cmd +++ b/conf/hbase-env.cmd @@ -27,6 +27,12 @@ @rem The maximum amount of heap to use, in MB. Default is 1000. @rem set HBASE_HEAPSIZE=1000 +@rem Uncomment below if you intend to use off heap cache. +@rem set HBASE_OFFHEAPSIZE=1000 + +@rem For example, to allocate 8G of offheap, to 8G: +@rem etHBASE_OFFHEAPSIZE=8G + @rem Extra Java runtime options. @rem Below are what we set by default. May only work with SUN JVM. @rem For more on why as well as other possible settings, diff --git a/conf/hbase-env.sh b/conf/hbase-env.sh index 2d10609b99c..1693b178460 100644 --- a/conf/hbase-env.sh +++ b/conf/hbase-env.sh @@ -34,6 +34,12 @@ # The maximum amount of heap to use, in MB. Default is 1000. # export HBASE_HEAPSIZE=1000 +# Uncomment below if you intend to use off heap cache. +# export HBASE_OFFHEAPSIZE=1000 + +# For example, to allocate 8G of offheap, to 8G: +# export HBASE_OFFHEAPSIZE=8G + # Extra Java runtime options. # Below are what we set by default. May only work with SUN JVM. # For more on why as well as other possible settings, @@ -66,10 +72,6 @@ export HBASE_OPTS="-XX:+UseConcMarkSweepGC" # If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR . # export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc: -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=1 -XX:GCLogFileSize=512M" -# Uncomment below if you intend to use off heap cache. -# export HBASE_OPTS="$HBASE_OPTS -XX:MaxDirectMemorySize=SET_THIS_TO_HOW_MANY_GIGS_OF_OFFHEAP" -# For example, to allocate 8G of offheap, set SET_THIS_TO_HOW_MANY_GIGS_OF_OFFHEAP to 8G as in: -# export HBASE_OPTS="$HBASE_OPTS -XX:MaxDirectMemorySize=8G" # See the package documentation for org.apache.hadoop.hbase.io.hfile for other configurations # needed setting up off-heap block caching. diff --git a/src/main/docbkx/book.xml b/src/main/docbkx/book.xml index 181d0d9ab26..3976ae98f21 100644 --- a/src/main/docbkx/book.xml +++ b/src/main/docbkx/book.xml @@ -2398,12 +2398,12 @@ rs.close(); First, edit the RegionServer's hbase-env.sh and set - -XX:MaxDirectMemorySize to a value greater than the offheap size wanted, in + HBASE_OFFHEAPSIZE to a value greater than the offheap size wanted, in this case, 4 GB (expressed as 4G). Lets set it to 5G. That'll be 4G for our offheap cache and 1G for any other uses of offheap memory (there are other users of offheap memory other than BlockCache; e.g. DFSClient - in RegionServer can make use of offheap memory). - -XX:MaxDirectMemorySize=5G + in RegionServer can make use of offheap memory). See . + HBASE_OFFHEAPSIZE=5G Next, add the following configuration to the RegionServer's