HBASE-14067 bundle ruby files for hbase shell into a jar.

* removes some cruft from the hbase-shell pom that appears to be from coping the hbase-server pom long ago
* puts the ruby scripts into the hbase-shell jar following the guide from jruby for packaging
* removes hard coding the location of the implementation scripts from our runtime
* removes hard coding the load path for the implementation from the test code (leaves hard coding the test code location)
* provides a work around for a name conflict between our shell and the ruby stdlib shell.

closes #2515

Signed-off-by: Michael Stack <stack@apache.org>

bin/hbase

@@ -509,13 +509,22 @@ fi
 # figure out which class to run
 if [ "$COMMAND" = "shell" ] ; then
 	#find the hbase ruby sources
+  # assume we are in a binary install if lib/ruby exists
   if [ -d "$HBASE_HOME/lib/ruby" ]; then
-    HBASE_OPTS="$HBASE_OPTS -Dhbase.ruby.sources=$HBASE_HOME/lib/ruby"
+    # We want jruby to consume these things rather than our bootstrap script;
+    # jruby will look for the env variable 'JRUBY_OPTS'.
+    JRUBY_OPTS="${JRUBY_OPTS} -X+O"
+    export JRUBY_OPTS
+    # hbase-shell.jar contains a 'jar-bootstrap.rb'
+    # for more info see
+    # https://github.com/jruby/jruby/wiki/StandaloneJarsAndClasses#standalone-executable-jar-files
+    CLASS="org.jruby.JarBootstrapMain"
+  # otherwise assume we are running in a source checkout
   else
     HBASE_OPTS="$HBASE_OPTS -Dhbase.ruby.sources=$HBASE_HOME/hbase-shell/src/main/ruby"
+    CLASS="org.jruby.Main -X+O ${JRUBY_OPTS} ${HBASE_HOME}/hbase-shell/src/main/ruby/jar-bootstrap.rb"
   fi
   HBASE_OPTS="$HBASE_OPTS $HBASE_SHELL_OPTS"
-  CLASS="org.jruby.Main -X+O ${JRUBY_OPTS} ${HBASE_HOME}/bin/hirb.rb"
 elif [ "$COMMAND" = "hbck" ] ; then
   # Look for the -j /path/to/HBCK2.jar parameter. Else pass through to hbck.
   case "${1}" in

bin/hirb.rb

@@ -1,5 +1,3 @@
-#
-#
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -15,217 +13,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
+puts <<EOF
-# File passed to org.jruby.Main by bin/hbase.  Pollutes jirb with hbase imports
+  This file has been superceded by packaging our ruby files into a jar
-# and hbase  commands and then loads jirb.  Outputs a banner that tells user
+  and using jruby's bootstrapping to invoke them. If you need to
-# where to find help, shell version, and loads up a custom hirb.
+  source this file fo some reason it is now named 'jar-bootstrap.rb' and is
-#
+  located in the root of the file hbase-shell.jar and in the source tree at
-# In noninteractive mode, runs commands from stdin until completion or an error.
+  'hbase-shell/src/main/ruby'.
-# On success will exit with status 0, on any problem will exit non-zero. Callers
+EOF
-# should only rely on "not equal to 0", because the current error exit code of 1
-# will likely be updated to diffentiate e.g. invalid commands, incorrect args,
-# permissions, etc.
-
-# TODO: Interrupt a table creation or a connection to a bad master.  Currently
-# has to time out.  Below we've set down the retries for rpc and hbase but
-# still can be annoying (And there seem to be times when we'll retry for
-# ever regardless)
-# TODO: Add support for listing and manipulating catalog tables, etc.
-# TODO: Encoding; need to know how to go from ruby String to UTF-8 bytes
-
-# Run the java magic include and import basic HBase types that will help ease
-# hbase hacking.
-include Java
-
-# Some goodies for hirb. Should these be left up to the user's discretion?
-require 'irb/completion'
-require 'pathname'
-
-# Add the directory names in hbase.jruby.sources commandline option
-# to the ruby load path so I can load up my HBase ruby modules
-sources = java.lang.System.getProperty('hbase.ruby.sources')
-$LOAD_PATH.unshift Pathname.new(sources)
-
-#
-# FIXME: Switch args processing to getopt
-#
-# See if there are args for this shell. If any, read and then strip from ARGV
-# so they don't go through to irb.  Output shell 'usage' if user types '--help'
-cmdline_help = <<HERE # HERE document output as shell usage
-Usage: shell [OPTIONS] [SCRIPTFILE [ARGUMENTS]]
-
- -d | --debug            Set DEBUG log levels.
- -h | --help             This help.
- -n | --noninteractive   Do not run within an IRB session and exit with non-zero
-                         status on first error.
- --top-level-defs        Compatibility flag to export HBase shell commands onto
-                         Ruby's main object
- -Dkey=value             Pass hbase-*.xml Configuration overrides. For example, to
-                         use an alternate zookeeper ensemble, pass:
-                           -Dhbase.zookeeper.quorum=zookeeper.example.org
-                         For faster fail, pass the below and vary the values:
-                           -Dhbase.client.retries.number=7
-                           -Dhbase.ipc.client.connect.max.retries=3
-HERE
-
-# Takes configuration and an arg that is expected to be key=value format.
-# If c is empty, creates one and returns it
-def add_to_configuration(c, arg)
-  kv = arg.split('=')
-  kv.length == 2 || (raise "Expected parameter #{kv} in key=value format")
-  c = org.apache.hadoop.hbase.HBaseConfiguration.create if c.nil?
-  c.set(kv[0], kv[1])
-  c
-end
-
-found = []
-script2run = nil
-log_level = org.apache.log4j.Level::ERROR
-@shell_debug = false
-interactive = true
-top_level_definitions = false
-_configuration = nil
-D_ARG = '-D'.freeze
-while (arg = ARGV.shift)
-  if arg == '-h' || arg == '--help'
-    puts cmdline_help
-    exit
-  elsif arg == D_ARG
-    argValue = ARGV.shift || (raise "#{D_ARG} takes a 'key=value' parameter")
-    _configuration = add_to_configuration(_configuration, argValue)
-    found.push(arg)
-    found.push(argValue)
-  elsif arg.start_with? D_ARG
-    _configuration = add_to_configuration(_configuration, arg[2..-1])
-    found.push(arg)
-  elsif arg == '-d' || arg == '--debug'
-    log_level = org.apache.log4j.Level::DEBUG
-    $fullBackTrace = true
-    @shell_debug = true
-    found.push(arg)
-    puts 'Setting DEBUG log level...'
-  elsif arg == '-n' || arg == '--noninteractive'
-    interactive = false
-    found.push(arg)
-  elsif arg == '-r' || arg == '--return-values'
-    warn '[INFO] the -r | --return-values option is ignored. we always behave '\
-         'as though it was given.'
-    found.push(arg)
-  elsif arg == '--top-level-defs'
-    top_level_definitions = true
-  else
-    # Presume it a script. Save it off for running later below
-    # after we've set up some environment.
-    script2run = arg
-    found.push(arg)
-    # Presume that any other args are meant for the script.
-    break
-  end
-end
-
-# Delete all processed args
-found.each { |arg| ARGV.delete(arg) }
-# Make sure debug flag gets back to IRB
-ARGV.unshift('-d') if @shell_debug
-
-# Set logging level to avoid verboseness
-org.apache.log4j.Logger.getLogger('org.apache.zookeeper').setLevel(log_level)
-org.apache.log4j.Logger.getLogger('org.apache.hadoop.hbase').setLevel(log_level)
-
-# Require HBase now after setting log levels
-require 'hbase_constants'
-
-# Load hbase shell
-require 'shell'
-
-# Require formatter
-require 'shell/formatter'
-
-# Setup the HBase module.  Create a configuration.
-@hbase = _configuration.nil? ? Hbase::Hbase.new : Hbase::Hbase.new(_configuration)
-
-# Setup console
-@shell = Shell::Shell.new(@hbase, interactive)
-@shell.debug = @shell_debug
-
-##
-# Toggle shell debugging
-#
-# @return [Boolean] true if debug is turned on after updating the flag
-def debug
-  if @shell_debug
-    @shell_debug = false
-    conf.back_trace_limit = 0
-    log_level = org.apache.log4j.Level::ERROR
-  else
-    @shell_debug = true
-    conf.back_trace_limit = 100
-    log_level = org.apache.log4j.Level::DEBUG
-  end
-  org.apache.log4j.Logger.getLogger('org.apache.zookeeper').setLevel(log_level)
-  org.apache.log4j.Logger.getLogger('org.apache.hadoop.hbase').setLevel(log_level)
-  debug?
-end
-
-##
-# Print whether debug is on or off
-def debug?
-  puts "Debug mode is #{@shell_debug ? 'ON' : 'OFF'}\n\n"
-  nil
-end
-
-
-# For backwards compatibility, this will export all the HBase shell commands, constants, and
-# instance variables (@hbase and @shell) onto Ruby's top-level receiver object known as "main".
-@shell.export_all(self) if top_level_definitions
-
-# If script2run, try running it.  If we're in interactive mode, will go on to run the shell unless
-# script calls 'exit' or 'exit 0' or 'exit errcode'.
-require 'shell/hbase_loader'
-if script2run
-  ::Shell::Shell.exception_handler(!$fullBackTrace) { @shell.eval_io(Hbase::Loader.file_for_load(script2run), filename = script2run) }
-end
-
-# If we are not running interactively, evaluate standard input
-::Shell::Shell.exception_handler(!$fullBackTrace) { @shell.eval_io(STDIN) } unless interactive
-
-if interactive
-  # Output a banner message that tells users where to go for help
-  @shell.print_banner
-
-  require 'irb'
-  require 'irb/ext/change-ws'
-  require 'irb/hirb'
-
-  module IRB
-    # Override of the default IRB.start
-    def self.start(ap_path = nil)
-      $0 = File.basename(ap_path, '.rb') if ap_path
-
-      IRB.setup(ap_path)
-      @CONF[:IRB_NAME] = 'hbase'
-      @CONF[:AP_NAME] = 'hbase'
-      @CONF[:BACK_TRACE_LIMIT] = 0 unless $fullBackTrace
-
-      hirb = if @CONF[:SCRIPT]
-               HIRB.new(nil, @CONF[:SCRIPT])
-             else
-               HIRB.new
-             end
-
-      shl = TOPLEVEL_BINDING.receiver.instance_variable_get :'@shell'
-      hirb.context.change_workspace shl.get_workspace
-
-      @CONF[:IRB_RC].call(hirb.context) if @CONF[:IRB_RC]
-      # Storing our current HBase IRB Context as the main context is imperative for several reasons,
-      # including auto-completion.
-      @CONF[:MAIN_CONTEXT] = hirb.context
-
-      catch(:IRB_EXIT) do
-        hirb.eval_input
-      end
-    end
-  end
-
-  IRB.start
-end

hbase-assembly/src/main/assembly/client-components.xml

@@ -76,13 +76,6 @@
         <include>hbase-config.cmd</include>
       </includes>
     </fileSet>
-    <!-- Move the ruby code over -->
-    <fileSet>
-      <directory>${project.basedir}/../hbase-shell/src/main/ruby</directory>
-      <outputDirectory>lib/ruby</outputDirectory>
-      <fileMode>0644</fileMode>
-      <directoryMode>0755</directoryMode>
-    </fileSet>
     <!-- Include native libraries -->
     <fileSet>
       <directory>${project.basedir}/../hbase-server/target/native</directory>

hbase-assembly/src/main/assembly/components.xml

@@ -69,13 +69,6 @@
         <include>**/*.cmd</include>
       </includes>
     </fileSet>
-    <!-- Move the ruby code over -->
-    <fileSet>
-      <directory>${project.basedir}/../hbase-shell/src/main/ruby</directory>
-      <outputDirectory>lib/ruby</outputDirectory>
-      <fileMode>0644</fileMode>
-      <directoryMode>0755</directoryMode>
-    </fileSet>
     <!-- Move the webapps to the webapp dir -->
     <fileSet>
       <directory>${project.basedir}/../hbase-server/target/hbase-webapps</directory>

hbase-shell/pom.xml

@@ -30,15 +30,9 @@
   <name>Apache HBase - Shell</name>
   <description>Shell for HBase</description>
   <build>
-    <!-- Makes sure the resources get added before they are processed
-      by placing this first -->
     <resources>
-      <!-- Add the build webabpps to the classpth -->
       <resource>
-        <directory>${project.build.directory}</directory>
+        <directory>src/main/ruby</directory>
-        <includes>
-          <include>hbase-webapps/**</include>
-        </includes>
       </resource>
     </resources>
     <testResources>
@@ -50,38 +44,15 @@
       </testResource>
     </testResources>
     <plugins>
-      <!-- Run with -Dmaven.test.skip.exec=true to build -tests.jar without running
-        tests (this is needed for upstream projects whose tests need this jar simply for
-        compilation) -->
-      <plugin>
-        <!--Make it so assembly:single does nothing in here-->
-        <artifactId>maven-assembly-plugin</artifactId>
-        <configuration>
-          <skipAssembly>true</skipAssembly>
-        </configuration>
-      </plugin>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-jar-plugin</artifactId>
         <configuration>
           <archive>
             <manifest>
-              <mainClass>org/apache/hadoop/hbase/mapreduce/Driver</mainClass>
+              <mainClass>org.jruby.JarBootstrapMain</mainClass>
             </manifest>
           </archive>
-          <!-- Exclude these 2 packages, because their dependency _binary_ files
-            include the sources, and Maven 2.2 appears to add them to the sources to compile,
-            weird -->
-          <excludes>
-            <exclude>org/apache/jute/**</exclude>
-            <exclude>org/apache/zookeeper/**</exclude>
-            <exclude>**/*.jsp</exclude>
-            <exclude>hbase-site.xml</exclude>
-            <exclude>hdfs-site.xml</exclude>
-            <exclude>log4j.properties</exclude>
-            <exclude>mapred-queues.xml</exclude>
-            <exclude>mapred-site.xml</exclude>
-          </excludes>
         </configuration>
       </plugin>
       <!-- Make a jar and put the sources in the jar -->
@@ -89,27 +60,6 @@
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-source-plugin</artifactId>
       </plugin>
-      <!-- General ant tasks, bound to different build phases -->
-      <plugin>
-        <groupId>org.codehaus.mojo</groupId>
-        <artifactId>build-helper-maven-plugin</artifactId>
-        <executions>
-          <!-- Add the generated sources -->
-          <execution>
-            <id>jspcSource-packageInfo-source</id>
-            <phase>generate-sources</phase>
-            <goals>
-              <goal>add-source</goal>
-            </goals>
-            <configuration>
-              <sources>
-                <source>${project.build.directory}/generated-jamon</source>
-                <source>${project.build.directory}/generated-sources/java</source>
-              </sources>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
       <!-- General plugins -->
       <plugin>
         <groupId>net.revelc.code</groupId>

hbase-shell/src/main/ruby/hbase_shell.rb

@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Ruby has a stdlib named 'shell' so using "require 'shell'" does not
+# work if our shell implementation is not on the local filesystem.
+# this is the absolute path to our shell implementation when packaged
+# in a jar. The level of indirection provided by this file lets things
+# still behave the same as in earlier releases if folks unpackage the
+# jar contents onto the local filesystem if they need that for some
+# other reason.
+require 'uri:classloader:/shell.rb'

hbase-shell/src/main/ruby/jar-bootstrap.rb

@@ -0,0 +1,235 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# File passed to org.jruby.Main by bin/hbase.  Pollutes jirb with hbase imports
+# and hbase  commands and then loads jirb.  Outputs a banner that tells user
+# where to find help, shell version, and loads up a custom hirb.
+#
+# In noninteractive mode, runs commands from stdin until completion or an error.
+# On success will exit with status 0, on any problem will exit non-zero. Callers
+# should only rely on "not equal to 0", because the current error exit code of 1
+# will likely be updated to diffentiate e.g. invalid commands, incorrect args,
+# permissions, etc.
+
+# TODO: Interrupt a table creation or a connection to a bad master.  Currently
+# has to time out.  Below we've set down the retries for rpc and hbase but
+# still can be annoying (And there seem to be times when we'll retry for
+# ever regardless)
+# TODO: Add support for listing and manipulating catalog tables, etc.
+# TODO: Encoding; need to know how to go from ruby String to UTF-8 bytes
+
+# Run the java magic include and import basic HBase types that will help ease
+# hbase hacking.
+include Java
+
+# Some goodies for hirb. Should these be left up to the user's discretion?
+require 'irb/completion'
+require 'pathname'
+
+# Add the directory names in hbase.jruby.sources commandline option
+# to the ruby load path so I can load up my HBase ruby modules
+# in case we are trying to get them out of source instead of jar
+# packaging.
+sources = java.lang.System.getProperty('hbase.ruby.sources')
+unless sources.nil?
+  $LOAD_PATH.unshift Pathname.new(sources)
+end
+
+#
+# FIXME: Switch args processing to getopt
+#
+# See if there are args for this shell. If any, read and then strip from ARGV
+# so they don't go through to irb.  Output shell 'usage' if user types '--help'
+cmdline_help = <<HERE # HERE document output as shell usage
+Usage: shell [OPTIONS] [SCRIPTFILE [ARGUMENTS]]
+
+ -d | --debug            Set DEBUG log levels.
+ -h | --help             This help.
+ -n | --noninteractive   Do not run within an IRB session and exit with non-zero
+                         status on first error.
+ --top-level-defs        Compatibility flag to export HBase shell commands onto
+                         Ruby's main object
+ -Dkey=value             Pass hbase-*.xml Configuration overrides. For example, to
+                         use an alternate zookeeper ensemble, pass:
+                           -Dhbase.zookeeper.quorum=zookeeper.example.org
+                         For faster fail, pass the below and vary the values:
+                           -Dhbase.client.retries.number=7
+                           -Dhbase.ipc.client.connect.max.retries=3
+HERE
+
+# Takes configuration and an arg that is expected to be key=value format.
+# If c is empty, creates one and returns it
+def add_to_configuration(c, arg)
+  kv = arg.split('=')
+  kv.length == 2 || (raise "Expected parameter #{kv} in key=value format")
+  c = org.apache.hadoop.hbase.HBaseConfiguration.create if c.nil?
+  c.set(kv[0], kv[1])
+  c
+end
+
+found = []
+script2run = nil
+log_level = org.apache.log4j.Level::ERROR
+@shell_debug = false
+interactive = true
+top_level_definitions = false
+_configuration = nil
+D_ARG = '-D'.freeze
+while (arg = ARGV.shift)
+  if arg == '-h' || arg == '--help'
+    puts cmdline_help
+    exit
+  elsif arg == D_ARG
+    argValue = ARGV.shift || (raise "#{D_ARG} takes a 'key=value' parameter")
+    _configuration = add_to_configuration(_configuration, argValue)
+    found.push(arg)
+    found.push(argValue)
+  elsif arg.start_with? D_ARG
+    _configuration = add_to_configuration(_configuration, arg[2..-1])
+    found.push(arg)
+  elsif arg == '-d' || arg == '--debug'
+    log_level = org.apache.log4j.Level::DEBUG
+    $fullBackTrace = true
+    @shell_debug = true
+    found.push(arg)
+    puts 'Setting DEBUG log level...'
+  elsif arg == '-n' || arg == '--noninteractive'
+    interactive = false
+    found.push(arg)
+  elsif arg == '-r' || arg == '--return-values'
+    warn '[INFO] the -r | --return-values option is ignored. we always behave '\
+         'as though it was given.'
+    found.push(arg)
+  elsif arg == '--top-level-defs'
+    top_level_definitions = true
+  else
+    # Presume it a script. Save it off for running later below
+    # after we've set up some environment.
+    script2run = arg
+    found.push(arg)
+    # Presume that any other args are meant for the script.
+    break
+  end
+end
+
+# Delete all processed args
+found.each { |arg| ARGV.delete(arg) }
+# Make sure debug flag gets back to IRB
+ARGV.unshift('-d') if @shell_debug
+
+# Set logging level to avoid verboseness
+org.apache.log4j.Logger.getLogger('org.apache.zookeeper').setLevel(log_level)
+org.apache.log4j.Logger.getLogger('org.apache.hadoop.hbase').setLevel(log_level)
+
+# Require HBase now after setting log levels
+require 'hbase_constants'
+
+# Load hbase shell
+require 'hbase_shell'
+
+# Require formatter
+require 'shell/formatter'
+
+# Setup the HBase module.  Create a configuration.
+@hbase = _configuration.nil? ? Hbase::Hbase.new : Hbase::Hbase.new(_configuration)
+
+# Setup console
+@shell = Shell::Shell.new(@hbase, interactive)
+@shell.debug = @shell_debug
+
+##
+# Toggle shell debugging
+#
+# @return [Boolean] true if debug is turned on after updating the flag
+def debug
+  if @shell_debug
+    @shell_debug = false
+    conf.back_trace_limit = 0
+    log_level = org.apache.log4j.Level::ERROR
+  else
+    @shell_debug = true
+    conf.back_trace_limit = 100
+    log_level = org.apache.log4j.Level::DEBUG
+  end
+  org.apache.log4j.Logger.getLogger('org.apache.zookeeper').setLevel(log_level)
+  org.apache.log4j.Logger.getLogger('org.apache.hadoop.hbase').setLevel(log_level)
+  debug?
+end
+
+##
+# Print whether debug is on or off
+def debug?
+  puts "Debug mode is #{@shell_debug ? 'ON' : 'OFF'}\n\n"
+  nil
+end
+
+
+# For backwards compatibility, this will export all the HBase shell commands, constants, and
+# instance variables (@hbase and @shell) onto Ruby's top-level receiver object known as "main".
+@shell.export_all(self) if top_level_definitions
+
+# If script2run, try running it.  If we're in interactive mode, will go on to run the shell unless
+# script calls 'exit' or 'exit 0' or 'exit errcode'.
+require 'shell/hbase_loader'
+if script2run
+  ::Shell::Shell.exception_handler(!$fullBackTrace) { @shell.eval_io(Hbase::Loader.file_for_load(script2run), filename = script2run) }
+end
+
+# If we are not running interactively, evaluate standard input
+::Shell::Shell.exception_handler(!$fullBackTrace) { @shell.eval_io(STDIN) } unless interactive
+
+if interactive
+  # Output a banner message that tells users where to go for help
+  @shell.print_banner
+
+  require 'irb'
+  require 'irb/ext/change-ws'
+  require 'irb/hirb'
+
+  module IRB
+    # Override of the default IRB.start
+    def self.start(ap_path = nil)
+      $0 = File.basename(ap_path, '.rb') if ap_path
+
+      IRB.setup(ap_path)
+      @CONF[:IRB_NAME] = 'hbase'
+      @CONF[:AP_NAME] = 'hbase'
+      @CONF[:BACK_TRACE_LIMIT] = 0 unless $fullBackTrace
+
+      hirb = if @CONF[:SCRIPT]
+               HIRB.new(nil, @CONF[:SCRIPT])
+             else
+               HIRB.new
+             end
+
+      shl = TOPLEVEL_BINDING.receiver.instance_variable_get :'@shell'
+      hirb.context.change_workspace shl.get_workspace
+
+      @CONF[:IRB_RC].call(hirb.context) if @CONF[:IRB_RC]
+      # Storing our current HBase IRB Context as the main context is imperative for several reasons,
+      # including auto-completion.
+      @CONF[:MAIN_CONTEXT] = hirb.context
+
+      catch(:IRB_EXIT) do
+        hirb.eval_input
+      end
+    end
+  end
+
+  IRB.start
+end

hbase-shell/src/test/java/org/apache/hadoop/hbase/client/AbstractTestShell.java

@@ -26,12 +26,19 @@ import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
 import org.apache.hadoop.hbase.security.access.SecureTestUtil;
 import org.apache.hadoop.hbase.security.visibility.VisibilityTestUtil;
+import org.jruby.embed.PathType;
 import org.jruby.embed.ScriptingContainer;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public abstract class AbstractTestShell {
 
+  private static final Logger LOG = LoggerFactory.getLogger(AbstractTestShell.class);
+
   protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
   protected final static ScriptingContainer jruby = new ScriptingContainer();
 
@@ -54,9 +61,8 @@ public abstract class AbstractTestShell {
   }
 
   protected static void setUpJRubyRuntime() {
-    // Configure jruby runtime
+    LOG.debug("Configure jruby runtime, cluster set to {}", TEST_UTIL);
     List<String> loadPaths = new ArrayList<>(2);
-    loadPaths.add("src/main/ruby");
     loadPaths.add("src/test/ruby");
     jruby.setLoadPaths(loadPaths);
     jruby.put("$TEST_CLUSTER", TEST_UTIL);
@@ -65,6 +71,34 @@ public abstract class AbstractTestShell {
     System.setProperty("jruby.native.verbose", "true");
   }
 
+  /**
+   * @return comma separated list of ruby script names for tests
+   */
+  protected String getIncludeList() {
+    return "";
+  }
+
+  /**
+   * @return comma separated list of ruby script names for tests to skip
+   */
+  protected String getExcludeList() {
+    return "";
+  }
+
+  @Test
+  public void testRunShellTests() throws IOException {
+    final String tests = getIncludeList();
+    final String excludes = getExcludeList();
+    if (!tests.isEmpty()) {
+      System.setProperty("shell.test.include", tests);
+    }
+    if (!excludes.isEmpty()) {
+      System.setProperty("shell.test.exclude", excludes);
+    }
+    LOG.info("Starting ruby tests. includes: {} excludes: {}", tests, excludes);
+    jruby.runScriptlet(PathType.ABSOLUTE, "src/test/ruby/tests_runner.rb");
+  }
+
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
     setUpConfig();

hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestAdminShell.java

@@ -33,10 +33,8 @@ public class TestAdminShell extends AbstractTestShell {
   public static final HBaseClassTestRule CLASS_RULE =
     HBaseClassTestRule.forClass(TestAdminShell.class);
 
-  @Test
+  @Override
-  public void testRunShellTests() throws IOException {
+  protected String getIncludeList() {
-    System.setProperty("shell.test.include", "admin_test.rb");
+    return "admin_test.rb";
-    // Start all ruby tests
-    jruby.runScriptlet(PathType.ABSOLUTE, "src/test/ruby/tests_runner.rb");
   }
 }

hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestAdminShell2.java

@@ -33,10 +33,8 @@ public class TestAdminShell2 extends AbstractTestShell {
   public static final HBaseClassTestRule CLASS_RULE =
     HBaseClassTestRule.forClass(TestAdminShell2.class);
 
-  @Test
+  @Override
-  public void testRunShellTests() throws IOException {
+  protected String getIncludeList() {
-    System.setProperty("shell.test.include", "admin2_test.rb");
+    return "admin2_test.rb";
-    // Start all ruby tests
-    jruby.runScriptlet(PathType.ABSOLUTE, "src/test/ruby/tests_runner.rb");
   }
 }

hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestQuotasShell.java

@@ -33,10 +33,8 @@ public class TestQuotasShell extends AbstractTestShell {
   public static final HBaseClassTestRule CLASS_RULE =
     HBaseClassTestRule.forClass(TestQuotasShell.class);
 
-  @Test
+  @Override
-  public void testRunShellTests() throws IOException {
+  protected String getIncludeList() {
-    System.setProperty("shell.test.include", "quotas_test.rb");
+    return "quotas_test.rb";
-    // Start all ruby tests
-    jruby.runScriptlet(PathType.ABSOLUTE, "src/test/ruby/tests_runner.rb");
   }
 }

hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestRSGroupShell.java

@@ -47,10 +47,8 @@ public class TestRSGroupShell extends AbstractTestShell {
     setUpJRubyRuntime();
   }
 
-  @Test
+  @Override
-  public void testRunShellTests() throws IOException {
+  protected String getIncludeList() {
-    System.setProperty("shell.test.include", "rsgroup_shell_test.rb");
+    return "rsgroup_shell_test.rb";
-    // Start all ruby tests
-    jruby.runScriptlet(PathType.ABSOLUTE, "src/test/ruby/tests_runner.rb");
   }
 }

hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestReplicationShell.java

@@ -33,10 +33,8 @@ public class TestReplicationShell extends AbstractTestShell {
   public static final HBaseClassTestRule CLASS_RULE =
       HBaseClassTestRule.forClass(TestReplicationShell.class);
 
-  @Test
+  @Override
-  public void testRunShellTests() throws IOException {
+  protected String getIncludeList() {
-    System.setProperty("shell.test.include", "replication_admin_test.rb");
+    return "replication_admin_test.rb";
-    // Start all ruby tests
-    jruby.runScriptlet(PathType.ABSOLUTE, "src/test/ruby/tests_runner.rb");
   }
 }

hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestShell.java

@@ -32,11 +32,9 @@ public class TestShell extends AbstractTestShell {
   @ClassRule
   public static final HBaseClassTestRule CLASS_RULE = HBaseClassTestRule.forClass(TestShell.class);
 
-  @Test
+  @Override
-  public void testRunShellTests() throws IOException {
+  protected String getExcludeList() {
-    System.setProperty("shell.test.exclude", "replication_admin_test.rb,rsgroup_shell_test.rb," +
+    return "replication_admin_test.rb,rsgroup_shell_test.rb,admin_test.rb,table_test.rb," +
-      "admin_test.rb,table_test.rb,quotas_test.rb,admin2_test.rb");
+        "quotas_test.rb,admin2_test.rb";
-    // Start all ruby tests
-    jruby.runScriptlet(PathType.ABSOLUTE, "src/test/ruby/tests_runner.rb");
   }
 }

hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestShellNoCluster.java

@@ -30,8 +30,12 @@ import org.junit.ClassRule;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 @Category({ ClientTests.class, MediumTests.class })
 public class TestShellNoCluster extends AbstractTestShell {
+  private static final Logger LOG = LoggerFactory.getLogger(TestShellNoCluster.class);
 
   @ClassRule
   public static final HBaseClassTestRule CLASS_RULE =
@@ -41,7 +45,6 @@ public class TestShellNoCluster extends AbstractTestShell {
   public static void setUpBeforeClass() throws Exception {
     // no cluster
     List<String> loadPaths = new ArrayList<>(2);
-    loadPaths.add("src/main/ruby");
     loadPaths.add("src/test/ruby");
     jruby.setLoadPaths(loadPaths);
     jruby.put("$TEST_CLUSTER", TEST_UTIL);
@@ -55,9 +58,11 @@ public class TestShellNoCluster extends AbstractTestShell {
     // no cluster
   }
 
+  // Keep the same name so we override the with-a-cluster test
+  @Override
   @Test
-  public void testRunNoClusterShellTests() throws IOException {
+  public void testRunShellTests() throws IOException {
-    // Start ruby tests without cluster
+    LOG.info("Start ruby tests without cluster");
-    jruby.runScriptlet(PathType.ABSOLUTE, "src/test/ruby/no_cluster_tests_runner.rb");
+    jruby.runScriptlet(PathType.CLASSPATH, "no_cluster_tests_runner.rb");
   }
 }

hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestTableShell.java

@@ -33,10 +33,8 @@ public class TestTableShell extends AbstractTestShell {
   public static final HBaseClassTestRule CLASS_RULE =
     HBaseClassTestRule.forClass(TestTableShell.class);
 
-  @Test
+  @Override
-  public void testRunShellTests() throws IOException {
+  protected String getIncludeList() {
-    System.setProperty("shell.test.include", "table_test.rb");
+    return "test_table.rb";
-    // Start all ruby tests
-    jruby.runScriptlet(PathType.ABSOLUTE, "src/test/ruby/tests_runner.rb");
   }
 }

hbase-shell/src/test/rsgroup/org/apache/hadoop/hbase/client/rsgroup/TestShellRSGroups.java

@@ -52,11 +52,9 @@ public class TestShellRSGroups {
   final Logger LOG = LoggerFactory.getLogger(getClass());
   private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
   private final static ScriptingContainer jruby = new ScriptingContainer();
-  private static String basePath;
 
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
-    basePath = System.getProperty("basedir");
 
     // Start mini cluster
     TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
@@ -80,8 +78,7 @@ public class TestShellRSGroups {
 
     // Configure jruby runtime
     List<String> loadPaths = new ArrayList<>(2);
-    loadPaths.add(basePath+"/src/main/ruby");
+    loadPaths.add("src/test/ruby");
-    loadPaths.add(basePath+"/src/test/ruby");
     jruby.setLoadPaths(loadPaths);
     jruby.put("$TEST_CLUSTER", TEST_UTIL);
     System.setProperty("jruby.jit.logging.verbose", "true");
@@ -99,8 +96,7 @@ public class TestShellRSGroups {
     try {
       // Start only GroupShellTest
       System.setProperty("shell.test", "Hbase::RSGroupShellTest");
-      jruby.runScriptlet(PathType.ABSOLUTE,
+      jruby.runScriptlet(PathType.ABSOLUTE, "src/test/ruby/tests_runner.rb");
-          basePath + "/src/test/ruby/tests_runner.rb");
     } finally {
       System.clearProperty("shell.test");
     }

hbase-shell/src/test/ruby/hbase/admin2_test.rb

@@ -17,7 +17,7 @@
 # limitations under the License.
 #
 
-require 'shell'
+require 'hbase_shell'
 require 'stringio'
 require 'hbase_constants'
 require 'hbase/hbase'

hbase-shell/src/test/ruby/hbase/admin_test.rb

@@ -17,7 +17,7 @@
 # limitations under the License.
 #
 
-require 'shell'
+require 'hbase_shell'
 require 'stringio'
 require 'hbase_constants'
 require 'hbase/hbase'

hbase-shell/src/test/ruby/hbase/list_regions_test_no_cluster.rb

@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-require 'shell'
+require 'hbase_shell'
 require 'hbase_constants'
 
 java_import 'org.apache.hadoop.hbase.HRegionLocation'

hbase-shell/src/test/ruby/hbase/quotas_test.rb

@@ -17,7 +17,7 @@
 # limitations under the License.
 #
 
-require 'shell'
+require 'hbase_shell'
 require 'stringio'
 require 'hbase_constants'
 require 'hbase/hbase'

hbase-shell/src/test/ruby/hbase/quotas_test_no_cluster.rb

@@ -17,7 +17,7 @@
 # limitations under the License.
 #
 
-require 'shell'
+require 'hbase_shell'
 require 'stringio'
 require 'hbase_constants'
 require 'hbase/hbase'

hbase-shell/src/test/ruby/hbase/replication_admin_test.rb

@@ -17,7 +17,7 @@
 # limitations under the License.
 #
 
-require 'shell'
+require 'hbase_shell'
 require 'hbase_constants'
 require 'hbase/hbase'
 require 'hbase/table'

hbase-shell/src/test/ruby/hbase/security_admin_test.rb

@@ -17,7 +17,7 @@
 # limitations under the License.
 #
 
-require 'shell'
+require 'hbase_shell'
 require 'hbase_constants'
 require 'hbase/hbase'
 require 'hbase/table'

hbase-shell/src/test/ruby/hbase/test_connection_no_cluster.rb

@@ -17,7 +17,7 @@
 # limitations under the License.
 #
 
-require 'shell'
+require 'hbase_shell'
 require 'stringio'
 require 'hbase_constants'
 require 'hbase/hbase'

hbase-shell/src/test/ruby/hbase/visibility_labels_admin_test.rb

@@ -17,7 +17,7 @@
 # limitations under the License.
 #
 
-require 'shell'
+require 'hbase_shell'
 require 'hbase_constants'
 require 'hbase/hbase'
 require 'hbase/table'

hbase-shell/src/test/ruby/shell/commands_test.rb

@@ -19,7 +19,7 @@
 
 require 'hbase_constants'
 require 'hbase/table'
-require 'shell'
+require 'hbase_shell'
 
 ##
 # Tests whether all registered commands have a help and command method

hbase-shell/src/test/ruby/shell/converter_test.rb

@@ -15,7 +15,7 @@
 # limitations under the License.
 
 require 'hbase_constants'
-require 'shell'
+require 'hbase_shell'
 
 module Hbase
   class ConverterTest < Test::Unit::TestCase

hbase-shell/src/test/ruby/shell/list_locks_test.rb

@@ -18,7 +18,7 @@
 #
 
 require 'hbase_constants'
-require 'shell'
+require 'hbase_shell'
 
 module Hbase
   class ListLocksTest < Test::Unit::TestCase

hbase-shell/src/test/ruby/shell/list_procedures_test.rb

@@ -18,7 +18,7 @@
 #
 
 require 'hbase_constants'
-require 'shell'
+require 'hbase_shell'
 
 module Hbase
   class ListProceduresTest < Test::Unit::TestCase

hbase-shell/src/test/ruby/shell/noninteractive_test.rb

@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 require 'hbase_constants'
-require 'shell'
+require 'hbase_shell'
 
 class NonInteractiveTest < Test::Unit::TestCase
   def setup

hbase-shell/src/test/ruby/shell/rsgroup_shell_test.rb

@@ -18,7 +18,7 @@
 #
 
 require 'hbase_constants'
-require 'shell'
+require 'hbase_shell'
 
 module Hbase
   class RSGroupShellTest < Test::Unit::TestCase

hbase-shell/src/test/ruby/shell/shell_test.rb

@@ -18,7 +18,7 @@
 #
 
 require 'hbase_constants'
-require 'shell'
+require 'hbase_shell'
 
 class ShellTest < Test::Unit::TestCase
   include Hbase::TestHelpers

hbase-shell/src/test/ruby/test_helper.rb

@@ -39,7 +39,7 @@ module Hbase
   module TestHelpers
     require 'hbase_constants'
     require 'hbase/hbase'
-    require 'shell'
+    require 'hbase_shell'
 
     def setup_hbase
       hbase = ::Hbase::Hbase.new($TEST_CLUSTER.getConfiguration)
@@ -169,7 +169,3 @@ end
 
 # Extend standard unit tests with our helpers
 Test::Unit::TestCase.extend(Testing::Declarative)
-
-# Add the $HBASE_HOME/lib/ruby directory to the ruby
-# load path so I can load up my HBase ruby modules
-$LOAD_PATH.unshift File.join(File.dirname(__FILE__), "..", "..", "main", "ruby")

hbase-shell/src/test/ruby/tests_runner.rb

@@ -21,6 +21,8 @@ require 'rubygems'
 require 'rake'
 require 'set'
 
+puts "Ruby description: #{RUBY_DESCRIPTION}"
+
 unless defined?($TEST_CLUSTER)
   include Java
 
@@ -68,6 +70,7 @@ files.each do |file|
     next
   end
   begin
+    puts "loading test file '#{filename}'."
     load(file)
   rescue => e
     puts "ERROR: #{e}"