From b58c15a6d75cee6bf73c1bc3bd24a63dd146e07f Mon Sep 17 00:00:00 2001 From: Michael Stack Date: Fri, 13 Jun 2008 22:42:11 +0000 Subject: [PATCH] HBASE-487 Replace hql w/ a hbase-friendly jirb or jython shell This commit finishes up the DDL M src/test/org/apache/hadoop/hbase/TestBloomFilters.java Changed name of default constants. M src/java/org/apache/hadoop/hbase/HColumnDescriptor.java Changed name of default constants. (getNameAsString): Added. M src/java/org/apache/hadoop/hbase/master/ModifyColumn.java toString column family byte array name. M bin/HBase.rb (alter, hcd): Added. M bin/hirb.rb More help doc. (alter, admin): Added. git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@667684 13f79535-47bb-0310-9956-ffa450edef68 --- bin/HBase.rb | 55 +++++++-- bin/hirb.rb | 70 ++++++----- conf/hbase-env.sh | 1 + .../hadoop/hbase/HColumnDescriptor.java | 109 ++++++++++-------- .../hadoop/hbase/master/ModifyColumn.java | 4 +- .../hadoop/hbase/regionserver/HStoreFile.java | 2 +- .../apache/hadoop/hbase/TestBloomFilters.java | 8 +- 7 files changed, 152 insertions(+), 97 deletions(-) diff --git a/bin/HBase.rb b/bin/HBase.rb index 2cf49bd2291..60e15446825 100644 --- a/bin/HBase.rb +++ b/bin/HBase.rb @@ -73,23 +73,54 @@ module HBase for arg in args raise TypeError.new(arg.class.to_s + " of " + arg.to_s + " is not of Hash type") \ unless arg.instance_of? Hash - name = arg[NAME] - raise ArgumentError.new("Column family " + arg + " must have a name at least") \ - unless name - # TODO: Process all other parameters for column family - # Check the family name for colon. Add it if missing. - index = name.index(':') - if not index - # Add a colon. If already a colon, its in the right place, - # or an exception will come up out of the addFamily - name << ':' - end - htd.addFamily(HColumnDescriptor.new(name)) + htd.addFamily(hcd(arg)) end @admin.createTable(htd) @formatter.header() @formatter.footer(now) end + + def alter(tableName, args) + now = Time.now + raise TypeError.new("Table name must be of type String") \ + unless tableName.instance_of? String + descriptor = hcd(args) + @admin.modifyColumn(tableName, descriptor.getNameAsString(), descriptor); + @formatter.header() + @formatter.footer(now) + end + + def hcd(arg) + # Return a new HColumnDescriptor made of passed args + # TODO: This is brittle code. + # Here is current HCD constructor: + # public HColumnDescriptor(final byte [] columnName, final int maxVersions, + # final CompressionType compression, final boolean inMemory, + # final boolean blockCacheEnabled, + # final int maxValueLength, final int timeToLive, + # BloomFilterDescriptor bloomFilter) + name = arg[NAME] + raise ArgumentError.new("Column family " + arg + " must have a name") \ + unless name + # Check the family name for colon. Add it if missing. + index = name.index(':') + if not index + # Add a colon. If already a colon, its in the right place, + # or an exception will come up out of the addFamily + name << ':' + end + # TODO: What encoding are Strings in jruby? + return HColumnDescriptor.new(name.to_java_bytes, + # JRuby uses longs for ints. Need to convert. Also constants are String + arg[MAX_VERSIONS]? arg[MAX_VERSIONS]: HColumnDescriptor::DEFAULT_MAX_VERSIONS, + arg[COMPRESSION]? HColumnDescriptor::CompressionType::valueOf(arg[COMPRESSION]): + HColumnDescriptor::DEFAULT_COMPRESSION, + arg[IN_MEMORY]? arg[IN_MEMORY]: HColumnDescriptor::DEFAULT_IN_MEMORY, + arg[BLOCKCACHE]? arg[BLOCKCACHE]: HColumnDescriptor::DEFAULT_BLOCKCACHE, + arg[MAX_LENGTH]? arg[MAX_LENGTH]: HColumnDescriptor::DEFAULT_MAX_LENGTH, + arg[TTL]? arg[TTL]: HColumnDescriptor::DEFAULT_TTL, + arg[BLOOMFILTER]? arg[BLOOMFILTER]: HColumnDescriptor::DEFAULT_BLOOMFILTER) + end end class Table diff --git a/bin/hirb.rb b/bin/hirb.rb index e6a91bed8aa..6cf971d897f 100644 --- a/bin/hirb.rb +++ b/bin/hirb.rb @@ -71,12 +71,13 @@ end # Promote hbase constants to be constants of this module so can # be used bare as keys in 'create', 'alter', etc. To see constants -# in IRB, type 'Object.constants'. +# in IRB, type 'Object.constants'. Don't promote defaults because +# flattens all types to String. Can be confusing. def promoteConstants(constants) # The constants to import are all in uppercase for c in constants if c == c.upcase - eval("%s = \"%s\"" % [c, c]) + eval("%s = \"%s\"" % [c, c]) unless c =~ /DEFAULT_.*/ end end end @@ -95,6 +96,15 @@ def help # TODO: Add help to the commands themselves rather than keep it distinct h = < alter 't1', {NAME => 'f1', MAX_VERSIONS => 5} + create Create table; pass a table name, a dictionary of specifications per column family, and optionally, named parameters of table options. Dictionaries are described below in the GENERAL NOTES section. Named @@ -102,9 +112,12 @@ HBASE SHELL COMMANDS: (constants) as keys and a '=>' key/value delimiter. Parameters are comma-delimited. For example, to create a table named 't1' with an alternate maximum region size and a single family named 'f1' with an - alternate maximum number of cells, type: + alternate maximum number of cells and 'record' compression, type: - create 't1' {NAME => 'f1', MAX_VERSIONS => 5}, REGION_SIZE => 123 + hbase> create 't1' {NAME => 'f1', MAX_VERSIONS => 5, \ + COMPRESSION => 'RECORD'}, REGION_SIZE => 1024 + + For compression types, pass one of 'NONE', 'RECORD', or 'BLOCK' describe Describe the named table. Outputs the table and family descriptors drop Drop the named table. Table must first be disabled @@ -119,15 +132,12 @@ GENERAL NOTES: Quote all names in the hbase shell such as table and column names. Don't forget commas delimiting command parameters. Dictionaries of configuration used in the creation and alteration of tables are ruby-style Hashes. They look like -this: - - { 'key1' => 'value1', 'key2' => 'value2', ...} - -They are opened and closed with curley-braces. Key/values are delimited by -the '=>' character combination. Usually keys are predefined constants that -do not need to be quoted such as NAME, MAX_VERSIONS, MAX_LENGTH, TTL, etc. -Type 'Object.constants' to see a (messy) list of all constants in the -environment. +this: { 'key1' => 'value1', 'key2' => 'value2', ...}. They are opened and +closed with curley-braces. Key/values are delimited by the '=>' character +combination. Usually keys are predefined constants such as NAME, MAX_VERSIONS, +COMPRESSION, MAX_LENGTH, TTL, etc. Constants do not need to be quoted. Type +'Object.constants' to see a (messy) list of all constants in the environment. +See http://wiki.apache.org/hadoop/Hbase/Shell for more on the HBase Shell. HERE puts h end @@ -141,45 +151,43 @@ end # DDL -def create(table_name, *args) +def admin() @admin = HBase::Admin.new(@configuration, @formatter) unless @admin - @admin.create(table_name, args) + @admin +end + +def create(table_name, *args) + admin().create(table_name, args) end def drop(table_name) - @admin = HBase::Admin.new(@configuration, @formatter) unless @admin - @admin.drop(table_name) + admin().drop(table_name) end -def alter(table_name, *args) - puts "Not implemented yet" +def alter(table_name, args) + admin().alter(table_name, args) end # Administration def list - @admin = HBase::Admin.new(@configuration, @formatter) unless @admin - @admin.list() + admin().list() end def describe(table_name) - @admin = HBase::Admin.new(@configuration, @formatter) unless @admin - @admin.describe(table_name) + admin().describe(table_name) end def enable(table_name) - @admin = HBase::Admin.new(@configuration, @formatter) unless @admin - @admin.enable(table_name) + admin().enable(table_name) end def disable(table_name) - @admin = HBase::Admin.new(@configuration, @formatter) unless @admin - @admin.disable(table_name) + admin().disable(table_name) end def exists(table_name) - @admin = HBase::Admin.new(@configuration, @formatter) unless @admin - @admin.exists(table_name) + admin().exists(table_name) end # CRUD @@ -201,7 +209,9 @@ def delete(table_name, row_key, *args) end # Output a banner message that tells users where to go for help -puts "HBase Shell; type 'help' for the list of supported HBase commands" +puts <' for list of supported commands. +HERE version require "irb" diff --git a/conf/hbase-env.sh b/conf/hbase-env.sh index fc2c214e4c1..dc7315f621d 100644 --- a/conf/hbase-env.sh +++ b/conf/hbase-env.sh @@ -23,6 +23,7 @@ # The java implementation to use. Required. # export JAVA_HOME=/usr/lib/j2sdk1.5-sun +export JAVA_HOME=/usr # Extra Java CLASSPATH elements. Optional. # export HBASE_CLASSPATH= diff --git a/src/java/org/apache/hadoop/hbase/HColumnDescriptor.java b/src/java/org/apache/hadoop/hbase/HColumnDescriptor.java index 5e9b6241054..fc63ef3c68b 100644 --- a/src/java/org/apache/hadoop/hbase/HColumnDescriptor.java +++ b/src/java/org/apache/hadoop/hbase/HColumnDescriptor.java @@ -55,43 +55,6 @@ public class HColumnDescriptor implements WritableComparable { /** Compress sequences of records together in blocks. */ BLOCK } - - /** - * Default compression type. - */ - public static final CompressionType DEFAULT_COMPRESSION_TYPE = - CompressionType.NONE; - - /** - * Default number of versions of a record to keep. - */ - public static final int DEFAULT_N_VERSIONS = 3; - - /** - * Default setting for whether to serve from memory or not. - */ - public static final boolean DEFAULT_IN_MEMORY = false; - - /** - * Default setting for whether to use a block cache or not. - */ - public static final boolean DEFAULT_BLOCK_CACHE_ENABLED = false; - - /** - * Default maximum length of cell contents. - */ - public static final int DEFAULT_MAX_VALUE_LENGTH = Integer.MAX_VALUE; - - /** - * Default time to live of cell contents. - */ - public static final int DEFAULT_TIME_TO_LIVE = HConstants.FOREVER; - - /** - * Default bloom filter description. - */ - public static final BloomFilterDescriptor DEFAULT_BLOOM_FILTER_DESCRIPTOR = - null; // Defines for jruby/shell public static final String NAME = "NAME"; @@ -104,24 +67,65 @@ public class HColumnDescriptor implements WritableComparable { public static final String BLOOMFILTER = "BLOOMFILTER"; public static final String FOREVER = "FOREVER"; + /** + * Default compression type. + */ + public static final CompressionType DEFAULT_COMPRESSION = + CompressionType.NONE; + + /** + * Default number of versions of a record to keep. + */ + public static final int DEFAULT_MAX_VERSIONS = 3; + + /** + * Default maximum cell length. + */ + public static final int DEFAULT_MAX_LENGTH = Integer.MAX_VALUE; + + /** + * Default setting for whether to serve from memory or not. + */ + public static final boolean DEFAULT_IN_MEMORY = false; + + /** + * Default setting for whether to use a block cache or not. + */ + public static final boolean DEFAULT_BLOCKCACHE = false; + + /** + * Default maximum length of cell contents. + */ + public static final int DEFAULT_MAX_VALUE_LENGTH = Integer.MAX_VALUE; + + /** + * Default time to live of cell contents. + */ + public static final int DEFAULT_TTL = HConstants.FOREVER; + + /** + * Default bloom filter description. + */ + public static final BloomFilterDescriptor DEFAULT_BLOOMFILTER = null; + // Column family name private byte [] name; // Number of versions to keep - private int maxVersions = DEFAULT_N_VERSIONS; + private int maxVersions = DEFAULT_MAX_VERSIONS; // Compression setting if any - private CompressionType compressionType = DEFAULT_COMPRESSION_TYPE; + private CompressionType compressionType = DEFAULT_COMPRESSION; // Serve reads from in-memory cache private boolean inMemory = DEFAULT_IN_MEMORY; // Serve reads from in-memory block cache - private boolean blockCacheEnabled = DEFAULT_BLOCK_CACHE_ENABLED; + private boolean blockCacheEnabled = DEFAULT_BLOCKCACHE; // Maximum value size - private int maxValueLength = Integer.MAX_VALUE; + private int maxValueLength = DEFAULT_MAX_LENGTH; // Time to live of cell contents, in seconds from last timestamp - private int timeToLive = HConstants.FOREVER; + private int timeToLive = DEFAULT_TTL; // True if bloom filter was specified private boolean bloomFilterSpecified = false; // Descriptor of bloom filter - private BloomFilterDescriptor bloomFilter = DEFAULT_BLOOM_FILTER_DESCRIPTOR; + private BloomFilterDescriptor bloomFilter = DEFAULT_BLOOMFILTER; /** * Default constructor. Must be present for Writable. @@ -159,10 +163,10 @@ public class HColumnDescriptor implements WritableComparable { public HColumnDescriptor(final byte [] columnName) { this (columnName == null || columnName.length <= 0? HConstants.EMPTY_BYTE_ARRAY: columnName, - DEFAULT_N_VERSIONS, DEFAULT_COMPRESSION_TYPE, DEFAULT_IN_MEMORY, - DEFAULT_BLOCK_CACHE_ENABLED, - Integer.MAX_VALUE, DEFAULT_TIME_TO_LIVE, - DEFAULT_BLOOM_FILTER_DESCRIPTOR); + DEFAULT_MAX_VERSIONS, DEFAULT_COMPRESSION, DEFAULT_IN_MEMORY, + DEFAULT_BLOCKCACHE, + Integer.MAX_VALUE, DEFAULT_TTL, + DEFAULT_BLOOMFILTER); } /** @@ -245,6 +249,13 @@ public class HColumnDescriptor implements WritableComparable { return name; } + /** + * @return Name of this column family + */ + public String getNameAsString() { + return Bytes.toString(this.name); + } + /** @return compression type being used for the column family */ public CompressionType getCompression() { return this.compressionType; @@ -302,8 +313,8 @@ public class HColumnDescriptor implements WritableComparable { public String toString() { return "{" + NAME + " => '" + Bytes.toString(name) + "', " + MAX_VERSIONS + " => " + maxVersions + - ", " + COMPRESSION + " => " + this.compressionType + - ", " + IN_MEMORY + " => " + inMemory + + ", " + COMPRESSION + " => '" + this.compressionType + + "', " + IN_MEMORY + " => " + inMemory + ", " + BLOCKCACHE + " => " + blockCacheEnabled + ", " + MAX_LENGTH + " => " + maxValueLength + ", " + TTL + " => " + diff --git a/src/java/org/apache/hadoop/hbase/master/ModifyColumn.java b/src/java/org/apache/hadoop/hbase/master/ModifyColumn.java index c577354b37b..2d1a1665ca5 100644 --- a/src/java/org/apache/hadoop/hbase/master/ModifyColumn.java +++ b/src/java/org/apache/hadoop/hbase/master/ModifyColumn.java @@ -24,6 +24,7 @@ import java.io.IOException; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.InvalidColumnNameException; import org.apache.hadoop.hbase.ipc.HRegionInterface; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.HRegionInfo; /** Instantiated to modify an existing column family on a table */ @@ -47,7 +48,8 @@ class ModifyColumn extends ColumnOperation { i.getTableDesc().addFamily(descriptor); updateRegionInfo(server, m.getRegionName(), i); } else { // otherwise, we have an error. - throw new InvalidColumnNameException("Column family '" + columnName + + throw new InvalidColumnNameException("Column family '" + + Bytes.toString(columnName) + "' doesn't exist, so cannot be modified."); } } diff --git a/src/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java b/src/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java index 0150ebce9cf..1f117c9dca3 100644 --- a/src/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java +++ b/src/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java @@ -990,4 +990,4 @@ public class HStoreFile implements HConstants { return super.seek(key); } } -} +} \ No newline at end of file diff --git a/src/test/org/apache/hadoop/hbase/TestBloomFilters.java b/src/test/org/apache/hadoop/hbase/TestBloomFilters.java index b9edc7703be..6f2d405ca6e 100644 --- a/src/test/org/apache/hadoop/hbase/TestBloomFilters.java +++ b/src/test/org/apache/hadoop/hbase/TestBloomFilters.java @@ -172,9 +172,9 @@ public class TestBloomFilters extends HBaseClusterTestCase { 1, // Max versions HColumnDescriptor.CompressionType.NONE, // no compression HColumnDescriptor.DEFAULT_IN_MEMORY, // not in memory - HColumnDescriptor.DEFAULT_BLOCK_CACHE_ENABLED, + HColumnDescriptor.DEFAULT_BLOCKCACHE, HColumnDescriptor.DEFAULT_MAX_VALUE_LENGTH, - HColumnDescriptor.DEFAULT_TIME_TO_LIVE, + HColumnDescriptor.DEFAULT_TTL, bloomFilter ) ); @@ -238,9 +238,9 @@ public class TestBloomFilters extends HBaseClusterTestCase { 1, // Max versions HColumnDescriptor.CompressionType.NONE, // no compression HColumnDescriptor.DEFAULT_IN_MEMORY, // not in memory - HColumnDescriptor.DEFAULT_BLOCK_CACHE_ENABLED, + HColumnDescriptor.DEFAULT_BLOCKCACHE, HColumnDescriptor.DEFAULT_MAX_VALUE_LENGTH, - HColumnDescriptor.DEFAULT_TIME_TO_LIVE, + HColumnDescriptor.DEFAULT_TTL, bloomFilter ) );