HBASE-3691 Add compressor support for 'snappy', google's compressor

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1125163 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -257,6 +257,8 @@ Release 0.91.0 - Unreleased
    HBASE-3837  Show regions in transition on the master web page (todd)
    HBASE-3839  Add monitoring of currently running tasks to the master and
                RS web UIs
+   HBASE-3691  Add compressor support for 'snappy', google's compressor
+               (Nichole Treadway and Nicholas Telford)
 
 Release 0.90.4 - Unreleased
 

src/docbkx/book.xml

@@ -1373,6 +1373,45 @@ false
     reports in your logs; see <xref linkend="brand.new.compressor" />).
     </para>
     </section>
+    <section xml:id="snappy.compression">
+    <title>
+    SNAPPY
+    </title>
+    <para>
+        To set snappy compression on a column family, do as following:
+        <orderedlist>
+            <listitem>
+                <para>
+                 Install hadoop-snappy using these instructions: http://code.google.com/p/hadoop-snappy/
+                </para>
+            </listitem>
+            <listitem>
+                <para>
+                    You need to ensure the hadoop-snappy libs (incl. the native libs) are in the HBase classpath. One way to do this is
+                    to just symlink the libs from <filename>HADOOP_HOME/lib</filename> to <filename>HBASE_HOME/lib</filename>.
+                    This needs to be done on all HBase nodes, as with LZO.
+                </para>
+            </listitem>
+            <listitem>
+                <para>
+        Use CompressionTest to verify snappy support is enabled and the libs can be loaded:
+        <programlisting>$ hbase org.apache.hadoop.hbase.util.CompressionTest hdfs://host/path/to/hbase snappy</programlisting>
+                </para>
+            </listitem>
+            <listitem>
+                <para>
+        Create a column family with snappy compression and verify it in the hbase shell:
+        <programlisting>$ hbase> create 't1', { NAME => 'cf1', COMPRESSION => 'snappy' }
+hbase> describe 't1'</programlisting>
+        In the output of the "describe" command, you need to ensure it lists "COMPRESSION => 'snappy'"
+                </para>
+            </listitem>
+
+        </orderedlist>
+
+    </para>
+    </section>
+
   </appendix>
 
   <appendix xml:id="faq">

src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java

@@ -432,6 +432,7 @@ public class HColumnDescriptor implements WritableComparable<HColumnDescriptor>
     switch (type) {
       case LZO: compressionType = "LZO"; break;
       case GZ: compressionType = "GZ"; break;
+      case SNAPPY: compressionType = "SNAPPY"; break;
       default: compressionType = "NONE"; break;
     }
     setValue(COMPRESSION, compressionType);
@@ -456,6 +457,7 @@ public class HColumnDescriptor implements WritableComparable<HColumnDescriptor>
     switch (type) {
       case LZO: compressionType = "LZO"; break;
       case GZ: compressionType = "GZ"; break;
+      case SNAPPY: compressionType = "SNAPPY"; break;
       default: compressionType = "NONE"; break;
     }
     setValue(COMPRESSION_COMPACT, compressionType);

src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java

@@ -140,6 +140,25 @@ public final class Compression {
 
         return downStream;
       }
+    },
+    SNAPPY("snappy") {
+        // Use base type to avoid compile-time dependencies.
+        private transient CompressionCodec snappyCodec;
+
+        @Override
+        CompressionCodec getCodec(Configuration conf) {
+          if (snappyCodec == null) {
+            try {
+              Class<?> externalCodec =
+                  ClassLoader.getSystemClassLoader().loadClass("org.apache.hadoop.io.compress.SnappyCodec");
+              snappyCodec = (CompressionCodec) ReflectionUtils.newInstance(externalCodec, 
+                  conf);
+            } catch (ClassNotFoundException e) {
+              throw new RuntimeException(e);
+            }
+          }
+          return snappyCodec;
+        }
     };
 
     private final Configuration conf;

src/main/java/org/apache/hadoop/hbase/util/CompressionTest.java

@@ -94,7 +94,7 @@ public class CompressionTest {
 
   public static void usage() {
     System.err.println(
-      "Usage: CompressionTest <path> none|gz|lzo\n" +
+      "Usage: CompressionTest <path> none|gz|lzo|snappy\n" +
       "\n" +
       "For example:\n" +
       "  hbase " + CompressionTest.class + " file:///tmp/testfile gz\n");

src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java

@@ -129,7 +129,7 @@ public class TestHFilePerformance extends TestCase {
    * @param fileType "HFile" or "SequenceFile"
    * @param keyLength
    * @param valueLength
-   * @param codecName "none", "lzo", "gz"
+   * @param codecName "none", "lzo", "gz", "snappy"
    * @param rows number of rows to be written.
    * @param writeMethod used for HFile only.
    * @param minBlockSize used for HFile only.

src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java

@@ -281,7 +281,7 @@ public class TestHFileSeek extends TestCase {
 
     private Options buildOptions() {
       Option compress =
-          OptionBuilder.withLongOpt("compress").withArgName("[none|lzo|gz]")
+          OptionBuilder.withLongOpt("compress").withArgName("[none|lzo|gz|snappy]")
               .hasArg().withDescription("compression scheme").create('c');
 
       Option fileSize =
@@ -446,7 +446,7 @@ public class TestHFileSeek extends TestCase {
 
     private void validateOptions() throws ParseException {
       if (!compress.equals("none") && !compress.equals("lzo")
-          && !compress.equals("gz")) {
+          && !compress.equals("gz") && !compress.equals("snappy")) {
         throw new ParseException("Unknown compression scheme: " + compress);
       }
 

src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java

@@ -437,6 +437,9 @@ public class TestHFileOutputFormat  {
     if (numCfs-- > 0) {
       familyToCompression.put("Family1!@#!@#&", Compression.Algorithm.LZO);
     }
+    if (numCfs-- > 0) {
+      familyToCompression.put("Family2=asdads&!AASD", Compression.Algorithm.SNAPPY);
+    }
     if (numCfs-- > 0) {
       familyToCompression.put("Family2=asdads&!AASD", Compression.Algorithm.GZ);
     }

src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java

@@ -54,5 +54,6 @@ public class TestCompressionTest {
     assertFalse(CompressionTest.testCompression("LZO"));
     assertTrue(CompressionTest.testCompression("NONE"));
     assertTrue(CompressionTest.testCompression("GZ"));
+    assertTrue(CompressionTest.testCompression("SNAPPY"));
   }
 }