From 1342ed9b5c11a286d114f18b912f73670ef310db Mon Sep 17 00:00:00 2001 From: Michael Stack Date: Wed, 18 Jun 2008 22:24:34 +0000 Subject: [PATCH] HBASE-487 New shell.... Add in special handling of .META. table that we used have in HQL (so it prints out the HRegionInfo pretty). Also allow making a scanner without specifying columns. M src/java/org/apache/hadoop/hbase/HTableDescriptor.java Allow getMetadata work if HTable is set against meta tables. Was failing on isLegalTableName if name was one of the catalog table names. Needed by shell. M src/java/org/apache/hadoop/hbase/client/HTable.java Comment. M bin/hbase Remove commented out line. M bin/HBase.rb Allow passing just a table name to scanner; let it figure out all families Added in the special handling of .META. table cells that we used have in HQL so we can see start/end row, etc. Added in extra testing. M bin/Formatter.rb Allow setting width of emitted table in console formatter M bin/hirb.rb Allow setting width of emitted table in console formatter Improved scanner help.: git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@669318 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES.txt | 1 + bin/Formatter.rb | 6 +- bin/HBase.rb | 86 ++++++++++++++++--- bin/hbase | 1 - bin/hirb.rb | 69 +++++++++------ .../apache/hadoop/hbase/HTableDescriptor.java | 17 +++- .../apache/hadoop/hbase/client/HTable.java | 1 + 7 files changed, 136 insertions(+), 45 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 6f178a4afc9..f7e1817bc1f 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -100,6 +100,7 @@ Hbase Change Log region state (Jean-Daniel Cryans via Stack) HBASE-639 Add HBaseAdmin.getTableDescriptor function HBASE-533 Region Historian + HBASE-487 Replace hql w/ a hbase-friendly jirb or jython shell Release 0.1.2 - 05/13/2008 diff --git a/bin/Formatter.rb b/bin/Formatter.rb index 1c872f7304a..785afac4f0e 100644 --- a/bin/Formatter.rb +++ b/bin/Formatter.rb @@ -3,7 +3,7 @@ module Formatter # Base abstract class for results formatting. class Formatter # Takes an output stream and a print width. - def initialize(o, w = 80) + def initialize(o, w = 100) raise TypeError.new("Type %s of parameter %s is not IO" % [o.class, o]) \ unless o.instance_of? IO @out = o @@ -38,8 +38,8 @@ module Formatter puts end elsif args.length == 2 - col1width = 8 - col2width = 70 + col1width = @maxWidth / 4 + col2width = @maxWidth - col1width - 2 splits1 = split(col1width, dump(args[0])) splits2 = split(col2width, dump(args[1])) biggest = (splits2.length > splits1.length)? splits2.length: splits1.length diff --git a/bin/HBase.rb b/bin/HBase.rb index 75e9fc553ea..eb4694c52f2 100644 --- a/bin/HBase.rb +++ b/bin/HBase.rb @@ -16,6 +16,8 @@ import org.apache.hadoop.hbase.io.Cell import org.apache.hadoop.hbase.HBaseConfiguration import org.apache.hadoop.hbase.HColumnDescriptor import org.apache.hadoop.hbase.HTableDescriptor +import org.apache.hadoop.hbase.util.Bytes +import org.apache.hadoop.hbase.util.Writables module HBase COLUMN = "COLUMN" @@ -25,6 +27,7 @@ module HBase VERSIONS = HConstants::VERSIONS STOPROW = "STOPROW" STARTROW = "STARTROW" + ENDROW = STOPROW LIMIT = "LIMIT" # Wrapper for org.apache.hadoop.hbase.client.HBaseAdmin @@ -199,10 +202,25 @@ module HBase @formatter.footer(now) end + def getAllColumns + htd = @table.getMetadata() + result = [] + for f in htd.getFamilies() + n = f.getNameAsString() + n << ':' + result << n + end + result + end + def scan(columns, args = {}) now = Time.now if not columns or columns.length < 1 - raise ArgumentError.new("Must supply an array of columns to scan") + # Make up list of columns. + columns = getAllColumns() + end + if columns.class == String + columns = [columns] end cs = columns.to_java(java.lang.String) limit = -1 @@ -221,15 +239,15 @@ module HBase end end count = 0 - @formatter.header(["Row", "Column+Cell"]) + @formatter.header(["ROW", "COLUMN+CELL"]) i = s.iterator() while i.hasNext() r = i.next() row = String.from_java_bytes r.getRow() for k, v in r column = String.from_java_bytes k - cell = v.toString() - @formatter.row([row, "column=%s, %s" % [column, v.toString()]]) + cell = toString(column, v) + @formatter.row([row, "column=%s, %s" % [column, cell]]) end count += 1 if limit != -1 and count >= limit @@ -252,6 +270,28 @@ module HBase @formatter.header() @formatter.footer(now) end + + def isMetaTable() + tn = @table.getTableName() + return Bytes.equals(tn, HConstants::META_TABLE_NAME) or + Bytes.equals(tn, HConstants::META_TABLE_NAME) + + end + + # Make a String of the passed cell. + # Intercept cells whose format we know such as the info:regioninfo in .META. + def toString(column, cell) + if isMetaTable() + if column == 'info:regioninfo' + hri = Writables.getHRegionInfoOrNull(cell.getValue()) + return "timestamp=%d, value=%s" % [cell.getTimestamp(), hri.toString()] + elsif column == 'info:serverstartcode' + return "timestamp=%d, value=%s" % [cell.getTimestamp(), \ + Bytes.toLong(cell.getValue())] + end + end + cell.toString() + end # Get from table def get(row, args = {}) @@ -293,12 +333,11 @@ module HBase h = nil if result.instance_of? RowResult h = String.from_java_bytes result.getRow() - @formatter.header(["Column", "Cell"]) + @formatter.header(["COLUMN", "CELL"]) if result - for column, cell in result - v = String.from_java_bytes cell.getValue() - ts = cell.getTimestamp() - @formatter.row([(String.from_java_bytes column), cell.toString()]) + for k, v in result + column = String.from_java_bytes k + @formatter.row([column, toString(column, v)]) end end else @@ -314,8 +353,8 @@ module HBase end end - # Do a bit of testing. - # To run this test, do: ./bin/hbase org.jruby.Main bin/HBase.rb + # Testing. To run this test, there needs to be an hbase cluster up and + # running. Then do: ${HBASE_HOME}/bin/hbase org.jruby.Main bin/HBase.rb if $0 == __FILE__ # Add this directory to LOAD_PATH; presumption is that Formatter module # sits beside this one. Then load it up. @@ -326,7 +365,8 @@ module HBase # Now add in java and hbase classes configuration = HBaseConfiguration.new() admin = Admin.new(configuration, formatter) - # Create a table; drop old one if exists first. + # Drop old table. If it does not exist, get an exception. Catch and + # continue TESTTABLE = "HBase_rb_testtable" begin admin.disable(TESTTABLE) @@ -338,13 +378,31 @@ module HBase # Presume it exists. If it doesn't, next items will fail. table = Table.new(configuration, TESTTABLE, formatter) for i in 1..10 - table.put('x', 'x:%d' % i, 'x%d' % i) + table.put('x%d' % i, 'x:%d' % i, 'x%d' % i) end - table.get('x', {COLUMN => 'x:1'}) + table.get('x1', {COLUMN => 'x:1'}) if formatter.rowCount() != 1 raise IOError.new("Failed first put") end table.scan(['x:']) + if formatter.rowCount() != 10 + raise IOError.new("Failed scan of expected 10 rows") + end + # Verify that limit works. + table.scan(['x:'], {LIMIT => 3}) + if formatter.rowCount() != 3 + raise IOError.new("Failed scan of expected 3 rows") + end + # Should only be two rows if we start at 8 (Row x10 sorts beside x1). + table.scan(['x:'], {STARTROW => 'x8', LIMIT => 3}) + if formatter.rowCount() != 2 + raise IOError.new("Failed scan of expected 2 rows") + end + # Scan between two rows + table.scan(['x:'], {STARTROW => 'x5', ENDROW => 'x8'}) + if formatter.rowCount() != 3 + raise IOError.new("Failed endrow test") + end admin.disable(TESTTABLE) admin.drop(TESTTABLE) end diff --git a/bin/hbase b/bin/hbase index 52e80a180cf..6ca95999745 100755 --- a/bin/hbase +++ b/bin/hbase @@ -183,7 +183,6 @@ unset IFS # figure out which class to run if [ "$COMMAND" = "shell" ] ; then - # CLASS="org.jruby.Main --command irb -r${HBASE_HOME}/bin/hirb.rb" CLASS="org.jruby.Main ${HBASE_HOME}/bin/hirb.rb" elif [ "$COMMAND" = "master" ] ; then CLASS='org.apache.hadoop.hbase.master.HMaster' diff --git a/bin/hirb.rb b/bin/hirb.rb index 87145227701..93b6617d433 100644 --- a/bin/hirb.rb +++ b/bin/hirb.rb @@ -5,9 +5,10 @@ # TODO: Add 'debug' support (client-side logs show in shell). Add it as # command-line option and as command. # TODO: Interrupt a table creation or a connection to a bad master. Currently -# has to time out. +# has to time out. Below we've set down the retries for rpc and hbase but +# still can be annoying (And there seem to be times when we'll retry for +# ever regardless) # TODO: Add support for listing and manipulating catalog tables, etc. -# TODO: Fix 'irb: warn: can't alias help from irb_help.' in banner message # TODO: Encoding; need to know how to go from ruby String to UTF-8 bytes # Run the java magic include and import basic HBase types that will help ease @@ -28,12 +29,14 @@ require 'HBase' # so they don't go through to irb. Output shell 'usage' if user types '--help' cmdline_help = < put 't1', 'r1', 'c1', ts1 - scan Scan a table; pass table name and an array of column names. - Optionally, pass a dictionary of options that includes one or more - of the following: LIMIT, FILTER, STARTROW, STOPROW, and TIMESTAMP. - For example, to scan column 'c1', and 'c2', in table 't1' returning - 10 rows only: - - hbase> scan 't1', ['c1', 'c2'], {LIMIT => 10} + scan Scan a table; pass table name and optionally an array of column + names and a dictionary of scanner specification that includes one + or more of following: LIMIT, FILTER, STARTROW, STOPROW, or TIMESTAMP. + Examples: + + hbase> scan '.META.' + hbase> scan '.META.', ['info:regioninfo'] + hbase> scan 't1', ['c1', 'c2'], {LIMIT => 10, STARTROW => 'xyz'} version Output this HBase version @@ -242,7 +253,7 @@ def put(table, row, column, value, timestamp = nil) table(table).put(row, column, value, timestamp) end -def scan(table, columns, args = {}) +def scan(table, columns = [], args = {}) table(table).scan(columns, args) end @@ -268,18 +279,27 @@ version require "irb" -# IRB::ExtendCommandBundle.instance_variable_get("@EXTEND_COMMANDS").delete_if{|x| x.first == :irb_help} - module IRB - module ExtendCommandBundle - # These are attempts at blocking the complaint about :irb_help on startup. - # @EXTEND_COMMANDS.delete_if{|x| x[0] == :irb_help} - # @EXTEND_COMMANDS.each{|x| x[3][1] = OVERRIDE_ALL if x[0] == :irb_help} - # @EXTEND_COMMANDS.each{|x| puts x if x[0] == :irb_help} - end - # Subclass of IRB so can intercept methods class HIRB < Irb + def initialize + # This is ugly. Our 'help' method above provokes the following message + # on irb construction: 'irb: warn: can't alias help from irb_help.' + # Below, we reset the output so its pointed at /dev/null during irb + # construction just so this message does not come out after we emit + # the banner. Other attempts at playing with the hash of methods + # down in IRB didn't seem to work. I think the worst thing that can + # happen is the shell exiting because of failed IRB construction with + # no error (though we're not blanking STDERR) + begin + f = File.open("/dev/null", "w") + $stdout = f + super + ensure + f.close() + $stdout = STDOUT + end + end def output_value # Suppress output if last_value is 'nil' @@ -295,7 +315,8 @@ module IRB $0 = File::basename(ap_path, ".rb") if ap_path IRB.setup(ap_path) - @CONF[:IRB_NAME]="hbase" + @CONF[:IRB_NAME] = 'hbase' + @CONF[:AP_NAME] = 'hbase' if @CONF[:SCRIPT] hirb = HIRB.new(nil, @CONF[:SCRIPT]) diff --git a/src/java/org/apache/hadoop/hbase/HTableDescriptor.java b/src/java/org/apache/hadoop/hbase/HTableDescriptor.java index 43c5cf2ddfa..959e0366ab8 100644 --- a/src/java/org/apache/hadoop/hbase/HTableDescriptor.java +++ b/src/java/org/apache/hadoop/hbase/HTableDescriptor.java @@ -71,8 +71,7 @@ public class HTableDescriptor implements WritableComparable { */ private HTableDescriptor(final byte [] name, HColumnDescriptor[] families) { this.name = name.clone(); - this.rootregion = Bytes.equals(name, HConstants.ROOT_TABLE_NAME); - this.metaregion = true; + setMetaFlags(name); for(HColumnDescriptor descriptor : families) { this.families.put(Bytes.mapKey(descriptor.getName()), descriptor); } @@ -108,9 +107,21 @@ public class HTableDescriptor implements WritableComparable { * @see HADOOP-1581 HBASE: Un-openable tablename bug */ public HTableDescriptor(final byte [] name) { - this.name = isLegalTableName(name); + setMetaFlags(name); + this.name = this.metaregion? name: isLegalTableName(name); this.nameAsString = Bytes.toString(this.name); } + + /* + * Set meta flags on this table. + * Called by constructors. + * @param name + */ + private void setMetaFlags(final byte [] name) { + this.rootregion = Bytes.equals(name, HConstants.ROOT_TABLE_NAME); + this.metaregion = + this.rootregion? true: Bytes.equals(name, HConstants.META_TABLE_NAME); + } /** * Check passed buffer is legal user-space table name. diff --git a/src/java/org/apache/hadoop/hbase/client/HTable.java b/src/java/org/apache/hadoop/hbase/client/HTable.java index e0ca921362e..ab88349b9b6 100644 --- a/src/java/org/apache/hadoop/hbase/client/HTable.java +++ b/src/java/org/apache/hadoop/hbase/client/HTable.java @@ -341,6 +341,7 @@ public class HTable { * @return table metadata * @throws IOException */ + // Why is this deprecated? What should be used instead? St.Ack @Deprecated public HTableDescriptor getMetadata() throws IOException { return this.connection.getHTableDescriptor(this.tableName);