HBASE-487 New shell....

Add in special handling of .META. table that we used have in HQL
(so it prints out the HRegionInfo pretty).  Also allow making a scanner
without specifying columns.

M  src/java/org/apache/hadoop/hbase/HTableDescriptor.java
   Allow getMetadata work if HTable is set against meta tables.
   Was failing on isLegalTableName if name was one of the catalog table names.
   Needed by shell.
M src/java/org/apache/hadoop/hbase/client/HTable.java
  Comment.
M  bin/hbase
  Remove commented out line.
M  bin/HBase.rb
  Allow passing just a table name to scanner; let it figure out all families
  Added in the special handling of .META. table cells that we used have in HQL
  so we can see start/end row, etc.  Added in extra testing.
M  bin/Formatter.rb
  Allow setting width of emitted table in console formatter
M  bin/hirb.rb
  Allow setting width of emitted table in console formatter
  Improved scanner help.:


git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@669318 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -100,6 +100,7 @@ Hbase Change Log
                region state (Jean-Daniel Cryans via Stack)
    HBASE-639   Add HBaseAdmin.getTableDescriptor function
    HBASE-533   Region Historian
+   HBASE-487   Replace hql w/ a hbase-friendly jirb or jython shell
    
 Release 0.1.2 - 05/13/2008
    

bin/Formatter.rb

@@ -3,7 +3,7 @@ module Formatter
   # Base abstract class for results formatting.
   class Formatter
     # Takes an output stream and a print width.
-    def initialize(o, w = 80)
+    def initialize(o, w = 100)
       raise TypeError.new("Type %s of parameter %s is not IO" % [o.class, o]) \
         unless o.instance_of? IO
       @out = o
@@ -38,8 +38,8 @@ module Formatter
           puts
         end
       elsif args.length == 2
-        col1width = 8
+        col1width = @maxWidth / 4
-        col2width = 70
+        col2width = @maxWidth - col1width - 2
         splits1 = split(col1width, dump(args[0]))
         splits2 = split(col2width, dump(args[1]))
         biggest = (splits2.length > splits1.length)? splits2.length: splits1.length

bin/HBase.rb

@@ -16,6 +16,8 @@ import org.apache.hadoop.hbase.io.Cell
 import org.apache.hadoop.hbase.HBaseConfiguration
 import org.apache.hadoop.hbase.HColumnDescriptor
 import org.apache.hadoop.hbase.HTableDescriptor
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.hadoop.hbase.util.Writables
 
 module HBase
   COLUMN = "COLUMN"
@@ -25,6 +27,7 @@ module HBase
   VERSIONS = HConstants::VERSIONS
   STOPROW = "STOPROW"
   STARTROW = "STARTROW"
+  ENDROW = STOPROW
   LIMIT = "LIMIT"
 
   # Wrapper for org.apache.hadoop.hbase.client.HBaseAdmin
@@ -199,10 +202,25 @@ module HBase
       @formatter.footer(now)
     end
 
+    def getAllColumns
+       htd = @table.getMetadata()
+       result = []
+       for f in htd.getFamilies()
+         n = f.getNameAsString()
+         n << ':'
+         result << n
+       end
+       result
+    end
+
     def scan(columns, args = {})
       now = Time.now 
       if not columns or columns.length < 1
-        raise ArgumentError.new("Must supply an array of columns to scan")
+        # Make up list of columns.
+        columns = getAllColumns()
+      end
+      if columns.class == String
+        columns = [columns]
       end
       cs = columns.to_java(java.lang.String)
       limit = -1
@@ -221,15 +239,15 @@ module HBase
         end
       end 
       count = 0
-      @formatter.header(["Row", "Column+Cell"])
+      @formatter.header(["ROW", "COLUMN+CELL"])
       i = s.iterator()
       while i.hasNext()
         r = i.next()
         row = String.from_java_bytes r.getRow()
         for k, v in r
           column = String.from_java_bytes k
-          cell = v.toString()
+          cell = toString(column, v)
-          @formatter.row([row, "column=%s, %s" % [column, v.toString()]])
+          @formatter.row([row, "column=%s, %s" % [column, cell]])
         end
         count += 1
         if limit != -1 and count >= limit
@@ -253,6 +271,28 @@ module HBase
       @formatter.footer(now)
     end
 
+    def isMetaTable()
+      tn = @table.getTableName()
+      return Bytes.equals(tn, HConstants::META_TABLE_NAME) or
+        Bytes.equals(tn, HConstants::META_TABLE_NAME)
+        
+    end
+
+    # Make a String of the passed cell.
+    # Intercept cells whose format we know such as the info:regioninfo in .META.
+    def toString(column, cell)
+      if isMetaTable()
+        if column == 'info:regioninfo'
+          hri = Writables.getHRegionInfoOrNull(cell.getValue())
+          return "timestamp=%d, value=%s" % [cell.getTimestamp(), hri.toString()]
+        elsif column == 'info:serverstartcode'
+          return "timestamp=%d, value=%s" % [cell.getTimestamp(), \
+            Bytes.toLong(cell.getValue())]
+        end
+      end
+      cell.toString()
+    end
+  
     # Get from table
     def get(row, args = {})
       now = Time.now 
@@ -293,12 +333,11 @@ module HBase
       h = nil
       if result.instance_of? RowResult
         h = String.from_java_bytes result.getRow()
-        @formatter.header(["Column", "Cell"])
+        @formatter.header(["COLUMN", "CELL"])
         if result
-          for column, cell in result
+          for k, v in result
-            v = String.from_java_bytes cell.getValue()
+            column = String.from_java_bytes k
-            ts = cell.getTimestamp()
+            @formatter.row([column, toString(column, v)])
-            @formatter.row([(String.from_java_bytes column), cell.toString()])
           end
         end
       else
@@ -314,8 +353,8 @@ module HBase
     end
   end
 
-  # Do a bit of testing.
+  # Testing. To run this test, there needs to be an hbase cluster up and
-  # To run this test, do: ./bin/hbase org.jruby.Main bin/HBase.rb
+  # running.  Then do: ${HBASE_HOME}/bin/hbase org.jruby.Main bin/HBase.rb
   if $0 == __FILE__
     # Add this directory to LOAD_PATH; presumption is that Formatter module
     # sits beside this one.  Then load it up.
@@ -326,7 +365,8 @@ module HBase
     # Now add in java and hbase classes
     configuration = HBaseConfiguration.new()
     admin = Admin.new(configuration, formatter)
-    # Create a table; drop old one if exists first.
+    # Drop old table.  If it does not exist, get an exception.  Catch and
+    # continue
     TESTTABLE = "HBase_rb_testtable"
     begin
       admin.disable(TESTTABLE)
@@ -338,13 +378,31 @@ module HBase
     # Presume it exists.  If it doesn't, next items will fail.
     table = Table.new(configuration, TESTTABLE, formatter) 
     for i in 1..10
-      table.put('x', 'x:%d' % i, 'x%d' % i)
+      table.put('x%d' % i, 'x:%d' % i, 'x%d' % i)
     end
-    table.get('x', {COLUMN => 'x:1'})
+    table.get('x1', {COLUMN => 'x:1'})
     if formatter.rowCount() != 1
       raise IOError.new("Failed first put")
     end
     table.scan(['x:'])
+    if formatter.rowCount() != 10
+      raise IOError.new("Failed scan of expected 10 rows")
+    end
+    # Verify that limit works.
+    table.scan(['x:'], {LIMIT => 3})
+    if formatter.rowCount() != 3
+      raise IOError.new("Failed scan of expected 3 rows")
+    end
+    # Should only be two rows if we start at 8 (Row x10 sorts beside x1).
+    table.scan(['x:'], {STARTROW => 'x8', LIMIT => 3})
+    if formatter.rowCount() != 2
+      raise IOError.new("Failed scan of expected 2 rows")
+    end
+    # Scan between two rows
+    table.scan(['x:'], {STARTROW => 'x5', ENDROW => 'x8'})
+    if formatter.rowCount() != 3
+      raise IOError.new("Failed endrow test")
+    end
     admin.disable(TESTTABLE)
     admin.drop(TESTTABLE)
   end

bin/hbase

@@ -183,7 +183,6 @@ unset IFS
 
 # figure out which class to run
 if [ "$COMMAND" = "shell" ] ; then
-  # CLASS="org.jruby.Main --command irb -r${HBASE_HOME}/bin/hirb.rb"
   CLASS="org.jruby.Main ${HBASE_HOME}/bin/hirb.rb"
 elif [ "$COMMAND" = "master" ] ; then
   CLASS='org.apache.hadoop.hbase.master.HMaster'

bin/hirb.rb

@@ -5,9 +5,10 @@
 # TODO: Add 'debug' support (client-side logs show in shell).  Add it as
 # command-line option and as command.
 # TODO: Interrupt a table creation or a connection to a bad master.  Currently
-# has to time out.
+# has to time out.  Below we've set down the retries for rpc and hbase but
+# still can be annoying (And there seem to be times when we'll retry for
+# ever regardless)
 # TODO: Add support for listing and manipulating catalog tables, etc.
-# TODO: Fix 'irb: warn: can't alias help from irb_help.' in banner message
 # TODO: Encoding; need to know how to go from ruby String to UTF-8 bytes
 
 # Run the java magic include and import basic HBase types that will help ease
@@ -29,11 +30,13 @@ require 'HBase'
 cmdline_help = <<HERE # HERE document output as shell usage
 HBase Shell command-line options:
  format        Formatter for outputting results: console | html. Default: console
+ format.width  Width of table outputs. Default: 110 characters.
  master        HBase master shell should connect to: e.g --master=example:60000
 HERE
 master = nil
-@formatter = Formatter::Console.new(STDOUT)
 found = []
+format = 'console'
+format_width = 110
 for arg in ARGV
   if arg =~ /^--master=(.+)/i
     master = $1
@@ -41,12 +44,16 @@ for arg in ARGV
   elsif arg =~ /^--format=(.+)/i
     format = $1
     if format =~ /^html$/i
-      @formatter = Formatter::XHTML.new(STDOUT)
+      raise NoMethodError.new("Not yet implemented")
     elsif format =~ /^console$/i
       # This is default
     else
       raise ArgumentError.new("Unsupported format " + arg)
     end
+    found.push(arg)
+  elsif arg =~ /^--format-width=(.+)/i
+    format_width = $1.to_i
+    found.push(arg)
   elsif arg == '-h' || arg == '--help'
     puts cmdline_help
     exit
@@ -55,6 +62,9 @@ end
 for arg in found
   ARGV.delete(arg)
 end
+# Presume console format.
+@formatter = Formatter::Console.new(STDOUT, format_width)
+# TODO, etc.  @formatter = Formatter::XHTML.new(STDOUT)
 
 # Setup the HBase module.  Create a configuration.  If a master, set it.
 # Turn off retries in hbase and ipc.  Human doesn't want to wait on N retries.
@@ -150,13 +160,14 @@ HBASE SHELL COMMANDS:
 
            hbase> put 't1', 'r1', 'c1', ts1
 
- scan      Scan a table; pass table name and an array of column names.
+ scan      Scan a table; pass table name and optionally an array of column
-           Optionally, pass a dictionary of options that includes one or more
+           names and a dictionary of scanner specification that includes one
-           of the following: LIMIT, FILTER, STARTROW, STOPROW, and TIMESTAMP.
+           or more of following: LIMIT, FILTER, STARTROW, STOPROW, or TIMESTAMP.
-           For example, to scan column 'c1', and 'c2', in table 't1' returning
+           Examples:
-           10 rows only:
            
-           hbase> scan 't1', ['c1', 'c2'], {LIMIT => 10}
+           hbase> scan '.META.'
+           hbase> scan '.META.', ['info:regioninfo']
+           hbase> scan 't1', ['c1', 'c2'], {LIMIT => 10, STARTROW => 'xyz'}
 
  version   Output this HBase version
 
@@ -242,7 +253,7 @@ def put(table, row, column, value, timestamp = nil)
   table(table).put(row, column, value, timestamp)
 end
   
-def scan(table, columns, args = {})
+def scan(table, columns = [], args = {})
   table(table).scan(columns, args)
 end
   
@@ -268,18 +279,27 @@ version
 
 require "irb"
 
-# IRB::ExtendCommandBundle.instance_variable_get("@EXTEND_COMMANDS").delete_if{|x| x.first == :irb_help}
-
 module IRB
-  module ExtendCommandBundle
-    # These are attempts at blocking the complaint about :irb_help on startup.
-    # @EXTEND_COMMANDS.delete_if{|x| x[0] == :irb_help}
-    # @EXTEND_COMMANDS.each{|x| x[3][1] = OVERRIDE_ALL if x[0] == :irb_help}
-    # @EXTEND_COMMANDS.each{|x| puts x if x[0] == :irb_help}
-  end
-
   # Subclass of IRB so can intercept methods
   class HIRB < Irb
+    def initialize
+      # This is ugly.  Our 'help' method above provokes the following message
+      # on irb construction: 'irb: warn: can't alias help from irb_help.'
+      # Below, we reset the output so its pointed at /dev/null during irb
+      # construction just so this message does not come out after we emit
+      # the banner.  Other attempts at playing with the hash of methods
+      # down in IRB didn't seem to work. I think the worst thing that can
+      # happen is the shell exiting because of failed IRB construction with
+      # no error (though we're not blanking STDERR)
+      begin
+        f = File.open("/dev/null", "w")
+        $stdout = f
+        super
+      ensure
+        f.close()
+        $stdout = STDOUT
+      end
+    end 
     
     def output_value
       # Suppress output if last_value is 'nil'
@@ -295,7 +315,8 @@ module IRB
     $0 = File::basename(ap_path, ".rb") if ap_path
 
     IRB.setup(ap_path)
-    @CONF[:IRB_NAME]="hbase"
+    @CONF[:IRB_NAME] = 'hbase'
+    @CONF[:AP_NAME] = 'hbase'
     
     if @CONF[:SCRIPT]
       hirb = HIRB.new(nil, @CONF[:SCRIPT])

src/java/org/apache/hadoop/hbase/HTableDescriptor.java

@@ -71,8 +71,7 @@ public class HTableDescriptor implements WritableComparable {
    */
   private HTableDescriptor(final byte [] name, HColumnDescriptor[] families) {
     this.name = name.clone();
-    this.rootregion = Bytes.equals(name, HConstants.ROOT_TABLE_NAME);
+    setMetaFlags(name);
-    this.metaregion = true;
     for(HColumnDescriptor descriptor : families) {
       this.families.put(Bytes.mapKey(descriptor.getName()), descriptor);
     }
@@ -108,10 +107,22 @@ public class HTableDescriptor implements WritableComparable {
    * @see <a href="HADOOP-1581">HADOOP-1581 HBASE: Un-openable tablename bug</a>
    */
   public HTableDescriptor(final byte [] name) {
-    this.name = isLegalTableName(name);
+    setMetaFlags(name);
+    this.name = this.metaregion? name: isLegalTableName(name);
     this.nameAsString = Bytes.toString(this.name);
   }
 
+  /*
+   * Set meta flags on this table.
+   * Called by constructors.
+   * @param name
+   */
+  private void setMetaFlags(final byte [] name) {
+    this.rootregion = Bytes.equals(name, HConstants.ROOT_TABLE_NAME);
+    this.metaregion =
+      this.rootregion? true: Bytes.equals(name, HConstants.META_TABLE_NAME);
+  }
+  
   /**
    * Check passed buffer is legal user-space table name.
    * @param b Table name.

src/java/org/apache/hadoop/hbase/client/HTable.java

@@ -341,6 +341,7 @@ public class HTable {
    * @return table metadata 
    * @throws IOException
    */
+  // Why is this deprecated?  What should be used instead? St.Ack
   @Deprecated
   public HTableDescriptor getMetadata() throws IOException {
     return this.connection.getHTableDescriptor(this.tableName);