diff --git a/bin/set_meta_memstore_size.rb b/bin/set_meta_memstore_size.rb new file mode 100644 index 00000000000..e340a5b46e2 --- /dev/null +++ b/bin/set_meta_memstore_size.rb @@ -0,0 +1,91 @@ +# +# Copyright 2011 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# This script must be used on a live cluster in order to fix .META.'s +# MEMSTORE_SIZE back to 64MB instead of the 16KB that was configured +# in 0.20 era. This is only required if .META. was created at that time. +# +# After running this script, HBase needs to be restarted. +# +# To see usage for this script, run: +# +# ${HBASE_HOME}/bin/hbase org.jruby.Main set_meta_memstore_size.rb +# +include Java +import org.apache.hadoop.hbase.util.Bytes +import org.apache.hadoop.hbase.HConstants +import org.apache.hadoop.hbase.HRegionInfo +import org.apache.hadoop.hbase.client.HTable +import org.apache.hadoop.hbase.client.Delete +import org.apache.hadoop.hbase.client.Put +import org.apache.hadoop.hbase.client.Scan +import org.apache.hadoop.hbase.HTableDescriptor +import org.apache.hadoop.hbase.HBaseConfiguration +import org.apache.hadoop.hbase.util.FSUtils +import org.apache.hadoop.hbase.util.Writables +import org.apache.hadoop.fs.Path +import org.apache.hadoop.fs.FileSystem +import org.apache.commons.logging.LogFactory + +# Name of this script +NAME = "set_meta_memstore_size.rb" + + +# Print usage for this script +def usage + puts 'Usage: %s.rb]' % NAME + exit! +end + +# Get configuration to use. +c = HBaseConfiguration.create() + +# Set hadoop filesystem configuration using the hbase.rootdir. +# Otherwise, we'll always use localhost though the hbase.rootdir +# might be pointing at hdfs location. +c.set("fs.default.name", c.get(HConstants::HBASE_DIR)) +fs = FileSystem.get(c) + +# Get a logger and a metautils instance. +LOG = LogFactory.getLog(NAME) + +# Check arguments +if ARGV.size > 0 + usage +end + +# Clean mentions of table from .META. +# Scan the .META. and remove all lines that begin with tablename +metaTable = HTable.new(c, HConstants::ROOT_TABLE_NAME) +scan = Scan.new() +scan.addColumn(HConstants::CATALOG_FAMILY, HConstants::REGIONINFO_QUALIFIER); +scanner = metaTable.getScanner(scan) +while (result = scanner.next()) + rowid = Bytes.toString(result.getRow()) + LOG.info("Settting memstore to 64MB on : " + rowid); + hriValue = result.getValue(HConstants::CATALOG_FAMILY, HConstants::REGIONINFO_QUALIFIER) + hri = Writables.getHRegionInfo(hriValue) + htd = hri.getTableDesc() + htd.setMemStoreFlushSize(64 * 1024 * 1024) + p = Put.new(result.getRow()) + p.add(HConstants::CATALOG_FAMILY, HConstants::REGIONINFO_QUALIFIER, Writables.getBytes(hri)); + metaTable.put(p) +end +scanner.close() diff --git a/src/docbkx/book.xml b/src/docbkx/book.xml index ed7d1900db8..da2f62c975b 100644 --- a/src/docbkx/book.xml +++ b/src/docbkx/book.xml @@ -288,34 +288,56 @@ Usually you'll want to use the latest version available except the problematic u
<link xlink:href="http://hadoop.apache.org">hadoop</link><indexterm><primary>Hadoop</primary></indexterm> This version of HBase will only run on Hadoop 0.20.x. It will not run on hadoop 0.21.x (nor 0.22.x) as of this writing. - HBase will lose data unless it is running on an HDFS that has a durable sync. - Currently only the branch-0.20-append - branch has this attribute. No official releases have been made from this branch as of this writing - so you will have to build your own Hadoop from the tip of this branch - Scroll down in the Hadoop How To Release to the section - Build Requirements for instruction on how to build Hadoop. - - or you could use - Cloudera's CDH3. - CDH has the 0.20-append patches needed to add a durable sync (As of this writing - CDH3 is still in beta. Either CDH3b2 or CDH3b3 will suffice). + HBase will lose data unless it is running on an HDFS that has a + durable sync. Currently only the + branch-0.20-append + branch has this attribute + + See CHANGES.txt - in branch-0.20-append to see list of patches involved. - Because HBase depends on Hadoop, it bundles an Hadoop instance under its lib directory. - The bundled Hadoop was made from the Apache branch-0.20-append branch. - If you want to run HBase on an Hadoop cluster that is other than a version made from branch-0.20.append, - you must replace the hadoop jar found in the HBase lib directory with the + in branch-0.20-append to see list of patches involved adding append on the Hadoop 0.20 branch. + + . + No official releases have been made from this branch up to now + so you will have to build your own Hadoop from the tip of this branch. + Scroll down in the Hadoop How To Release to the section + Build Requirements for instruction on how to build Hadoop. + + + + Or rather than build your own, you could use + Cloudera's CDH3. + CDH has the 0.20-append patches needed to add a durable sync (CDH3 is still in beta. + Either CDH3b2 or CDH3b3 will suffice). + + + Because HBase depends on Hadoop, it bundles an instance of + the Hadoop jar under its lib directory. + The bundled Hadoop was made from the Apache branch-0.20-append branch + at the time of this HBase's release. + It is critical that the version of Hadoop that is + out on your cluster matches what is Hbase match. Replace the hadoop + jar found in the HBase lib directory with the hadoop jar you are running out on your cluster to avoid version mismatch issues. + Make sure you replace the jar all over your cluster. For example, versions of CDH do not have HDFS-724 whereas Hadoops branch-0.20-append branch does have HDFS-724. This patch changes the RPC version because protocol was changed. Version mismatch issues have various manifestations but often all looks like its hung up. + + Can I just replace the jar in Hadoop 0.20.2 tarball with the <emphasis>sync</emphasis>-supporting Hadoop jar found in HBase? + + You could do this. It works going by a recent posting up on the + mailing list. + + Hadoop Security HBase will run on any Hadoop 0.20.x that incorporates Hadoop security features -- e.g. Y! 0.20S or CDH3B3 -- as long as you do as suggested above and replace the Hadoop jar that ships with HBase with the secure version. +
ssh ssh must be installed and sshd must @@ -903,8 +925,56 @@ index e70ebc6..96f8c27 100644
+ + + Upgrading + + Review the requirements + section above, in particular the section on Hadoop version. + +
+ Upgrading to HBase 0.90.x from 0.20.x or 0.89.x + This version of 0.90.x HBase can be started on data written by + HBase 0.20.x or HBase 0.89.x. There is no need of a migration step. + HBase 0.89.x and 0.90.x does write out the name of region directories + differently -- it names them with a md5 hash of the region name rather + than a jenkins hash -- so this means that once started, there is no + going back to HBase 0.20.x. + + + Be sure to remove the hbase-default.xml from + your conf + directory on upgrade. A 0.20.x version of this file will have + sub-optimal configurations for 0.90.x HBase. The + hbase-default.xml file is now bundled into the + HBase jar and read from there. If you would like to review + the content of this file, see it in the src tree at + src/main/resources/hbase-default.xml or + see Default HBase Configurations. + + + Finally, if upgrading from 0.20.x, check your + .META. schema in the shell. In the past we would + recommend that users run with a 16MB + MEMSTORE_SIZE. + Run hbase> scan '-ROOT-' in the shell. This will output + the current .META. schema. Check + MEMSTORE_SIZE size. Is it 16MB? If so, you will + need to change this. Run the script bin/set_meta_memstore_size.rb. + This will make the necessary edit to your .META. schema. + Failure to run this change will make for a slow cluster + + See HBASE-3499 Users upgrading to 0.90.0 need to have their .META. table updated with the right MEMSTORE_SIZE + + + . + + +
+ + Configuration @@ -2021,22 +2091,6 @@ When I build, why do I always get Unable to find resource 'VM_global_libra - - Upgrading your HBase - - - Whats involved upgrading to HBase 0.90.x from 0.89.x or from 0.20.x? - - - This version of 0.90.x HBase can be started on data written by - HBase 0.20.x or HBase 0.89.x. There is no need of a migration step. - HBase 0.89.x and 0.90.x does write out the name of region directories - differently -- it names them with a md5 hash of the region name rather - than a jenkins hash -- so this means that once started, there is no - going back to HBase 0.20.x. - - -