HADOOP-10693. Implementation of AES-CTR CryptoCodec using JNI to OpenSSL (hitliuyi via cmccabe)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/fs-encryption@1607768 13f79535-47bb-0310-9956-ffa450edef68

BUILDING.txt

@@ -81,6 +81,27 @@ Maven build goals:
     the final tar file. This option requires that -Dsnappy.lib is also given,
     and it ignores the -Dsnappy.prefix option.
 
+ OpenSSL build options:
+
+   OpenSSL includes a crypto library that can be utilized by the native code.
+   It is currently an optional component, meaning that Hadoop can be built with
+   or without this dependency.
+
+  * Use -Drequire.openssl to fail the build if libcrypto.so is not found.
+    If this option is not specified and the openssl library is missing,
+    we silently build a version of libhadoop.so that cannot make use of
+    openssl. This option is recommended if you plan on making use of openssl 
+    and want to get more repeatable builds.
+  * Use -Dopenssl.prefix to specify a nonstandard location for the libcrypto
+    header files and library files. You do not need this option if you have
+    installed openssl using a package manager.
+  * Use -Dopenssl.lib to specify a nonstandard location for the libcrypto library
+    files. Similarly to openssl.prefix, you do not need this option if you have
+    installed openssl using a package manager.
+  * Use -Dbundle.openssl to copy the contents of the openssl.lib directory into
+    the final tar file. This option requires that -Dopenssl.lib is also given,
+    and it ignores the -Dopenssl.prefix option.
+
    Tests options:
 
   * Use -DskipTests to skip tests when running the following Maven goals:

hadoop-common-project/hadoop-common/pom.xml

@@ -483,6 +483,10 @@
         <snappy.lib></snappy.lib>
         <snappy.include></snappy.include>
         <require.snappy>false</require.snappy>
+        <openssl.prefix></openssl.prefix>
+        <openssl.lib></openssl.lib>
+        <openssl.include></openssl.include>
+        <require.openssl>false</require.openssl>
       </properties>
       <build>
         <plugins>
@@ -532,6 +536,7 @@
                     <javahClassName>org.apache.hadoop.io.compress.snappy.SnappyDecompressor</javahClassName>
                     <javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Compressor</javahClassName>
                     <javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Decompressor</javahClassName>
+                    <javahClassName>org.apache.hadoop.crypto.OpensslCipher</javahClassName>
                     <javahClassName>org.apache.hadoop.util.NativeCrc32</javahClassName>
                     <javahClassName>org.apache.hadoop.net.unix.DomainSocket</javahClassName>
                     <javahClassName>org.apache.hadoop.net.unix.DomainSocketWatcher</javahClassName>
@@ -552,7 +557,7 @@
                 <configuration>
                   <target>
                     <exec executable="cmake" dir="${project.build.directory}/native" failonerror="true">
-                      <arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_BZIP2=${require.bzip2} -DREQUIRE_SNAPPY=${require.snappy} -DCUSTOM_SNAPPY_PREFIX=${snappy.prefix} -DCUSTOM_SNAPPY_LIB=${snappy.lib} -DCUSTOM_SNAPPY_INCLUDE=${snappy.include}"/>
+                      <arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_BZIP2=${require.bzip2} -DREQUIRE_SNAPPY=${require.snappy} -DCUSTOM_SNAPPY_PREFIX=${snappy.prefix} -DCUSTOM_SNAPPY_LIB=${snappy.lib} -DCUSTOM_SNAPPY_INCLUDE=${snappy.include} -DREQUIRE_OPENSSL=${require.openssl} -DCUSTOM_OPENSSL_PREFIX=${openssl.prefix} -DCUSTOM_OPENSSL_LIB=${openssl.lib} -DCUSTOM_OPENSSL_INCLUDE=${openssl.include}"/>
                     </exec>
                     <exec executable="make" dir="${project.build.directory}/native" failonerror="true">
                       <arg line="VERBOSE=1"/>
@@ -596,6 +601,11 @@
         <snappy.include></snappy.include>
         <require.snappy>false</require.snappy>
         <bundle.snappy.in.bin>true</bundle.snappy.in.bin>
+        <openssl.prefix></openssl.prefix>
+        <openssl.lib></openssl.lib>
+        <openssl.include></openssl.include>
+        <require.openssl>false</require.openssl>
+        <bundle.openssl.in.bin>true</bundle.openssl.in.bin>
       </properties>
       <build>
         <plugins>
@@ -641,6 +651,7 @@
                     <javahClassName>org.apache.hadoop.io.compress.snappy.SnappyDecompressor</javahClassName>
                     <javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Compressor</javahClassName>
                     <javahClassName>org.apache.hadoop.io.compress.lz4.Lz4Decompressor</javahClassName>
+                    <javahClassName>org.apache.hadoop.crypto.OpensslCipher</javahClassName>
                     <javahClassName>org.apache.hadoop.util.NativeCrc32</javahClassName>
                   </javahClassNames>
                   <javahOutputDirectory>${project.build.directory}/native/javah</javahOutputDirectory>
@@ -685,6 +696,10 @@
                     <argument>/p:CustomSnappyLib=${snappy.lib}</argument>
                     <argument>/p:CustomSnappyInclude=${snappy.include}</argument>
                     <argument>/p:RequireSnappy=${require.snappy}</argument>
+                    <argument>/p:CustomOpensslPrefix=${openssl.prefix}</argument>
+                    <argument>/p:CustomOpensslLib=${openssl.lib}</argument>
+                    <argument>/p:CustomOpensslInclude=${openssl.include}</argument>
+                    <argument>/p:RequireOpenssl=${require.openssl}</argument>
                   </arguments>
                 </configuration>
               </execution>

hadoop-common-project/hadoop-common/src/CMakeLists.txt

@@ -145,6 +145,37 @@ else (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
     ENDIF(REQUIRE_SNAPPY)
 endif (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
 
+SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES CMAKE_FIND_LIBRARY_SUFFIXES)
+set_find_shared_library_version("1.0.0")
+SET(OPENSSL_NAME "crypto")
+IF(${CMAKE_SYSTEM_NAME} MATCHES "Windows")
+    SET(OPENSSL_NAME "eay32")
+ENDIF()
+find_library(OPENSSL_LIBRARY
+    NAMES ${OPENSSL_NAME}
+    PATHS ${CUSTOM_OPENSSL_PREFIX} ${CUSTOM_OPENSSL_PREFIX}/lib
+          ${CUSTOM_OPENSSL_PREFIX}/lib64 ${CUSTOM_OPENSSL_LIB} NO_DEFAULT_PATH)
+find_library(OPENSSL_LIBRARY
+    NAMES ${OPENSSL_NAME})
+SET(CMAKE_FIND_LIBRARY_SUFFIXES STORED_CMAKE_FIND_LIBRARY_SUFFIXES)
+find_path(OPENSSL_INCLUDE_DIR 
+    NAMES openssl/evp.h
+    PATHS ${CUSTOM_OPENSSL_PREFIX} ${CUSTOM_OPENSSL_PREFIX}/include
+          ${CUSTOM_OPENSSL_INCLUDE} NO_DEFAULT_PATH)
+find_path(OPENSSL_INCLUDE_DIR 
+    NAMES openssl/evp.h)
+if (OPENSSL_LIBRARY AND OPENSSL_INCLUDE_DIR)
+    GET_FILENAME_COMPONENT(HADOOP_OPENSSL_LIBRARY ${OPENSSL_LIBRARY} NAME)
+    SET(OPENSSL_SOURCE_FILES
+        "${D}/crypto/OpensslCipher.c")
+else (OPENSSL_LIBRARY AND OPENSSL_INCLUDE_DIR)
+    SET(OPENSSL_INCLUDE_DIR "")
+    SET(OPENSSL_SOURCE_FILES "")
+    IF(REQUIRE_OPENSSL)
+        MESSAGE(FATAL_ERROR "Required openssl library could not be found.  OPENSSL_LIBRARY=${OPENSSL_LIBRARY}, OPENSSL_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR}, CUSTOM_OPENSSL_INCLUDE_DIR=${CUSTOM_OPENSSL_INCLUDE_DIR}, CUSTOM_OPENSSL_PREFIX=${CUSTOM_OPENSSL_PREFIX}, CUSTOM_OPENSSL_INCLUDE=${CUSTOM_OPENSSL_INCLUDE}")
+    ENDIF(REQUIRE_OPENSSL)
+endif (OPENSSL_LIBRARY AND OPENSSL_INCLUDE_DIR)
+
 include_directories(
     ${GENERATED_JAVAH}
     main/native/src
@@ -155,6 +186,7 @@ include_directories(
     ${ZLIB_INCLUDE_DIRS}
     ${BZIP2_INCLUDE_DIR}
     ${SNAPPY_INCLUDE_DIR}
+    ${OPENSSL_INCLUDE_DIR}
     ${D}/util
 )
 CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
@@ -172,6 +204,7 @@ add_dual_library(hadoop
     ${D}/io/compress/lz4/lz4.c
     ${D}/io/compress/lz4/lz4hc.c
     ${SNAPPY_SOURCE_FILES}
+    ${OPENSSL_SOURCE_FILES}
     ${D}/io/compress/zlib/ZlibCompressor.c
     ${D}/io/compress/zlib/ZlibDecompressor.c
     ${BZIP2_SOURCE_FILES}

hadoop-common-project/hadoop-common/src/config.h.cmake

@@ -21,6 +21,7 @@
 #cmakedefine HADOOP_ZLIB_LIBRARY "@HADOOP_ZLIB_LIBRARY@"
 #cmakedefine HADOOP_BZIP2_LIBRARY "@HADOOP_BZIP2_LIBRARY@"
 #cmakedefine HADOOP_SNAPPY_LIBRARY "@HADOOP_SNAPPY_LIBRARY@"
+#cmakedefine HADOOP_OPENSSL_LIBRARY "@HADOOP_OPENSSL_LIBRARY@"
 #cmakedefine HAVE_SYNC_FILE_RANGE
 #cmakedefine HAVE_POSIX_FADVISE
 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/AESCTRCryptoCodec.java

@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.crypto;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-import com.google.common.base.Preconditions;
-
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public abstract class AESCTRCryptoCodec extends CryptoCodec {
-
-  protected static final CipherSuite SUITE = CipherSuite.AES_CTR_NOPADDING;
-
-  /**
-   * For AES, the algorithm block is fixed size of 128 bits.
-   * @see http://en.wikipedia.org/wiki/Advanced_Encryption_Standard
-   */
-  private static final int AES_BLOCK_SIZE = SUITE.getAlgorithmBlockSize();
-  private static final int CTR_OFFSET = 8;
-
-  @Override
-  public CipherSuite getCipherSuite() {
-    return SUITE;
-  }
-  
-  /**
-   * The IV is produced by adding the initial IV to the counter. IV length 
-   * should be the same as {@link #AES_BLOCK_SIZE}
-   */
-  @Override
-  public void calculateIV(byte[] initIV, long counter, byte[] IV) {
-    Preconditions.checkArgument(initIV.length == AES_BLOCK_SIZE);
-    Preconditions.checkArgument(IV.length == AES_BLOCK_SIZE);
-    
-    System.arraycopy(initIV, 0, IV, 0, CTR_OFFSET);
-    long l = (initIV[CTR_OFFSET + 0] << 56)
-        + ((initIV[CTR_OFFSET + 1] & 0xFF) << 48)
-        + ((initIV[CTR_OFFSET + 2] & 0xFF) << 40)
-        + ((initIV[CTR_OFFSET + 3] & 0xFF) << 32)
-        + ((initIV[CTR_OFFSET + 4] & 0xFF) << 24)
-        + ((initIV[CTR_OFFSET + 5] & 0xFF) << 16)
-        + ((initIV[CTR_OFFSET + 6] & 0xFF) << 8)
-        + (initIV[CTR_OFFSET + 7] & 0xFF);
-    l += counter;
-    IV[CTR_OFFSET + 0] = (byte) (l >>> 56);
-    IV[CTR_OFFSET + 1] = (byte) (l >>> 48);
-    IV[CTR_OFFSET + 2] = (byte) (l >>> 40);
-    IV[CTR_OFFSET + 3] = (byte) (l >>> 32);
-    IV[CTR_OFFSET + 4] = (byte) (l >>> 24);
-    IV[CTR_OFFSET + 5] = (byte) (l >>> 16);
-    IV[CTR_OFFSET + 6] = (byte) (l >>> 8);
-    IV[CTR_OFFSET + 7] = (byte) (l);
-  }
-}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java

@@ -35,7 +35,7 @@ public abstract class CryptoCodec implements Configurable {
   
   public static CryptoCodec getInstance(Configuration conf) {
     final Class<? extends CryptoCodec> klass = conf.getClass(
-        HADOOP_SECURITY_CRYPTO_CODEC_CLASS_KEY, JCEAESCTRCryptoCodec.class, 
+        HADOOP_SECURITY_CRYPTO_CODEC_CLASS_KEY, JceAesCtrCryptoCodec.class, 
         CryptoCodec.class);
     return ReflectionUtils.newInstance(klass, conf);
   }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/JCEAESCTRCryptoCodec.java

@@ -1,159 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.crypto;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.security.GeneralSecurityException;
-import java.security.SecureRandom;
-
-import javax.crypto.Cipher;
-import javax.crypto.spec.IvParameterSpec;
-import javax.crypto.spec.SecretKeySpec;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-
-import com.google.common.base.Preconditions;
-
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY;
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_SECURE_RANDOM_ALGORITHM_KEY;
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_SECURE_RANDOM_ALGORITHM_DEFAULT;
-
-/**
- * Implement the AES-CTR crypto codec using JCE provider.
- */
-@InterfaceAudience.Private
-public class JCEAESCTRCryptoCodec extends AESCTRCryptoCodec {
-  private Configuration conf;
-  private String provider;
-  private SecureRandom random;
-
-  public JCEAESCTRCryptoCodec() {
-  }
-  
-  @Override
-  public Configuration getConf() {
-    return conf;
-  }
-  
-  @Override
-  public void setConf(Configuration conf) {
-    this.conf = conf;
-    provider = conf.get(HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY);
-    final String secureRandomAlg = conf.get(
-        HADOOP_SECURITY_SECURE_RANDOM_ALGORITHM_KEY, 
-        HADOOP_SECURITY_SECURE_RANDOM_ALGORITHM_DEFAULT);
-    try {
-      random = (provider != null) ? 
-          SecureRandom.getInstance(secureRandomAlg, provider) : 
-            SecureRandom.getInstance(secureRandomAlg);
-    } catch (GeneralSecurityException e) {
-      throw new IllegalArgumentException(e);
-    }
-  }
-
-  @Override
-  public Encryptor createEncryptor() throws GeneralSecurityException {
-    return new JCEAESCTRCipher(Cipher.ENCRYPT_MODE, provider);
-  }
-
-  @Override
-  public Decryptor createDecryptor() throws GeneralSecurityException {
-    return new JCEAESCTRCipher(Cipher.DECRYPT_MODE, provider);
-  }
-  
-  @Override
-  public void generateSecureRandom(byte[] bytes) {
-    random.nextBytes(bytes);
-  }  
-  
-  private static class JCEAESCTRCipher implements Encryptor, Decryptor {
-    private final Cipher cipher;
-    private final int mode;
-    private boolean contextReset = false;
-    
-    public JCEAESCTRCipher(int mode, String provider) 
-        throws GeneralSecurityException {
-      this.mode = mode;
-      if (provider == null || provider.isEmpty()) {
-        cipher = Cipher.getInstance(SUITE.getName());
-      } else {
-        cipher = Cipher.getInstance(SUITE.getName(), provider);
-      }
-    }
-
-    @Override
-    public void init(byte[] key, byte[] iv) throws IOException {
-      Preconditions.checkNotNull(key);
-      Preconditions.checkNotNull(iv);
-      contextReset = false;
-      try {
-        cipher.init(mode, new SecretKeySpec(key, "AES"), 
-            new IvParameterSpec(iv));
-      } catch (Exception e) {
-        throw new IOException(e);
-      }
-    }
-
-    /**
-     * AES-CTR will consume all of the input data. It requires enough space in 
-     * the destination buffer to encrypt entire input buffer.
-     */
-    @Override
-    public void encrypt(ByteBuffer inBuffer, ByteBuffer outBuffer)
-        throws IOException {
-      process(inBuffer, outBuffer);
-    }
-    
-    /**
-     * AES-CTR will consume all of the input data. It requires enough space in
-     * the destination buffer to decrypt entire input buffer.
-     */
-    @Override
-    public void decrypt(ByteBuffer inBuffer, ByteBuffer outBuffer)
-        throws IOException {
-      process(inBuffer, outBuffer);
-    }
-    
-    private void process(ByteBuffer inBuffer, ByteBuffer outBuffer)
-        throws IOException {
-      try {
-        int inputSize = inBuffer.remaining();
-        // Cipher#update will maintain crypto context.
-        int n = cipher.update(inBuffer, outBuffer);
-        if (n < inputSize) {
-          /**
-           * Typically code will not get here. Cipher#update will consume all 
-           * input data and put result in outBuffer. 
-           * Cipher#doFinal will reset the crypto context.
-           */
-          contextReset = true;
-          cipher.doFinal(inBuffer, outBuffer);
-        }
-      } catch (Exception e) {
-        throw new IOException(e);
-      }
-    }
-    
-    @Override
-    public boolean isContextReset() {
-      return contextReset;
-    }
-  }
-}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java

@@ -79,6 +79,11 @@ public class NativeCodeLoader {
    */
   public static native boolean buildSupportsSnappy();
   
+  /**
+   * Returns true only if this build was compiled with support for openssl.
+   */
+  public static native boolean buildSupportsOpenssl();
+
   public static native String getLibraryName();
 
   /**

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java

@@ -20,6 +20,7 @@ package org.apache.hadoop.util;
 
 import org.apache.hadoop.util.NativeCodeLoader;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.OpensslCipher;
 import org.apache.hadoop.io.compress.Lz4Codec;
 import org.apache.hadoop.io.compress.SnappyCodec;
 import org.apache.hadoop.io.compress.bzip2.Bzip2Factory;
@@ -57,12 +58,14 @@ public class NativeLibraryChecker {
     boolean nativeHadoopLoaded = NativeCodeLoader.isNativeCodeLoaded();
     boolean zlibLoaded = false;
     boolean snappyLoaded = false;
+    boolean opensslLoaded = false;
     // lz4 is linked within libhadoop
     boolean lz4Loaded = nativeHadoopLoaded;
     boolean bzip2Loaded = Bzip2Factory.isNativeBzip2Loaded(conf);
     String hadoopLibraryName = "";
     String zlibLibraryName = "";
     String snappyLibraryName = "";
+    String opensslLibraryName = "";
     String lz4LibraryName = "";
     String bzip2LibraryName = "";
     if (nativeHadoopLoaded) {
@@ -76,6 +79,11 @@ public class NativeLibraryChecker {
       if (snappyLoaded && NativeCodeLoader.buildSupportsSnappy()) {
         snappyLibraryName = SnappyCodec.getLibraryName();
       }
+      opensslLoaded = NativeCodeLoader.buildSupportsOpenssl() &&
+          OpensslCipher.isNativeCodeLoaded();
+      if (opensslLoaded) {
+        opensslLibraryName = OpensslCipher.getLibraryName();
+      }
       if (lz4Loaded) {
         lz4LibraryName = Lz4Codec.getLibraryName();
       }
@@ -89,6 +97,7 @@ public class NativeLibraryChecker {
     System.out.printf("snappy:  %b %s\n", snappyLoaded, snappyLibraryName);
     System.out.printf("lz4:     %b %s\n", lz4Loaded, lz4LibraryName);
     System.out.printf("bzip2:   %b %s\n", bzip2Loaded, bzip2LibraryName);
+    System.out.printf("openssl: %b %s\n", opensslLoaded, opensslLibraryName);
     if ((!nativeHadoopLoaded) ||
         (checkAll && !(zlibLoaded && snappyLoaded && lz4Loaded && bzip2Loaded))) {
       // return 1 to indicated check failed

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c

@@ -39,6 +39,16 @@ JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_util_NativeCodeLoader_buildSup
 #endif
 }
 
+JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_util_NativeCodeLoader_buildSupportsOpenssl
+  (JNIEnv *env, jclass clazz)
+{
+#ifdef HADOOP_OPENSSL_LIBRARY
+  return JNI_TRUE;
+#else
+  return JNI_FALSE;
+#endif
+}
+
 JNIEXPORT jstring JNICALL Java_org_apache_hadoop_util_NativeCodeLoader_getLibraryName
   (JNIEnv *env, jclass clazz)
 {

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java

@@ -17,38 +17,165 @@
  */
 package org.apache.hadoop.crypto;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.BufferedInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.junit.AfterClass;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.RandomDatum;
+import org.apache.hadoop.util.NativeCodeLoader;
+import org.apache.hadoop.util.ReflectionUtils;
 import org.junit.Assert;
-import org.junit.BeforeClass;
 import org.junit.Test;
 
 public class TestCryptoCodec {
-  private static CryptoCodec codec;
+  private static final Log LOG= LogFactory.getLog(TestCryptoCodec.class);
+  private static final byte[] key = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 
+    0x07, 0x08, 0x09, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16};
+  private static final byte[] iv = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 
+    0x07, 0x08, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08};
+  private static final int bufferSize = 4096;
   
-  @BeforeClass
+  private Configuration conf = new Configuration();
-  public static void init() throws Exception {
+  private int count = 10000;
-    Configuration conf = new Configuration();
+  private int seed = new Random().nextInt();
-    codec = CryptoCodec.getInstance(conf);
-  }
-  
-  @AfterClass
-  public static void shutdown() throws Exception {
-  }
   
   @Test(timeout=120000)
-  public void testSecureRandom() throws Exception {
+  public void testJceAesCtrCryptoCodec() throws Exception {
-    // len = 16
+    cryptoCodecTest(conf, seed, 0, 
-    checkSecureRandom(16);
+        "org.apache.hadoop.crypto.JceAesCtrCryptoCodec");
-    
+    cryptoCodecTest(conf, seed, count, 
-    // len = 32
+        "org.apache.hadoop.crypto.JceAesCtrCryptoCodec");
-    checkSecureRandom(32);
-    
-    // len = 128
-    checkSecureRandom(128);
   }
   
-  private void checkSecureRandom(int len) {
+  @Test(timeout=1200000)
+  public void testOpensslAesCtrCryptoCodec() throws Exception {
+    if (NativeCodeLoader.buildSupportsOpenssl()) {
+      Assert.assertTrue(OpensslCipher.isNativeCodeLoaded());
+    }
+    if (OpensslCipher.isNativeCodeLoaded()) {
+      cryptoCodecTest(conf, seed, 0, 
+          "org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec");
+      cryptoCodecTest(conf, seed, count, 
+          "org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec");
+    }
+  }
+  
+  private void cryptoCodecTest(Configuration conf, int seed, int count, 
+      String codecClass) throws IOException, GeneralSecurityException {
+    CryptoCodec codec = null;
+    try {
+      codec = (CryptoCodec)ReflectionUtils.newInstance(
+          conf.getClassByName(codecClass), conf);
+    } catch (ClassNotFoundException cnfe) {
+      throw new IOException("Illegal crypto codec!");
+    }
+    LOG.info("Created a Codec object of type: " + codecClass);
+    
+    // Generate data
+    DataOutputBuffer data = new DataOutputBuffer();
+    RandomDatum.Generator generator = new RandomDatum.Generator(seed);
+    for(int i = 0; i < count; ++i) {
+      generator.next();
+      RandomDatum key = generator.getKey();
+      RandomDatum value = generator.getValue();
+      
+      key.write(data);
+      value.write(data);
+    }
+    LOG.info("Generated " + count + " records");
+    
+    // Encrypt data
+    DataOutputBuffer encryptedDataBuffer = new DataOutputBuffer();
+    CryptoOutputStream out = new CryptoOutputStream(encryptedDataBuffer, 
+        codec, bufferSize, key, iv);
+    out.write(data.getData(), 0, data.getLength());
+    out.flush();
+    out.close();
+    LOG.info("Finished encrypting data");
+    
+    // Decrypt data
+    DataInputBuffer decryptedDataBuffer = new DataInputBuffer();
+    decryptedDataBuffer.reset(encryptedDataBuffer.getData(), 0, 
+        encryptedDataBuffer.getLength());
+    CryptoInputStream in = new CryptoInputStream(decryptedDataBuffer, 
+        codec, bufferSize, key, iv);
+    DataInputStream dataIn = new DataInputStream(new BufferedInputStream(in));
+    
+    // Check
+    DataInputBuffer originalData = new DataInputBuffer();
+    originalData.reset(data.getData(), 0, data.getLength());
+    DataInputStream originalIn = new DataInputStream(
+        new BufferedInputStream(originalData));
+    
+    for(int i=0; i < count; ++i) {
+      RandomDatum k1 = new RandomDatum();
+      RandomDatum v1 = new RandomDatum();
+      k1.readFields(originalIn);
+      v1.readFields(originalIn);
+      
+      RandomDatum k2 = new RandomDatum();
+      RandomDatum v2 = new RandomDatum();
+      k2.readFields(dataIn);
+      v2.readFields(dataIn);
+      assertTrue("original and encrypted-then-decrypted-output not equal",
+                 k1.equals(k2) && v1.equals(v2));
+      
+      // original and encrypted-then-decrypted-output have the same hashCode
+      Map<RandomDatum, String> m = new HashMap<RandomDatum, String>();
+      m.put(k1, k1.toString());
+      m.put(v1, v1.toString());
+      String result = m.get(k2);
+      assertEquals("k1 and k2 hashcode not equal", result, k1.toString());
+      result = m.get(v2);
+      assertEquals("v1 and v2 hashcode not equal", result, v1.toString());
+    }
+
+    // Decrypt data byte-at-a-time
+    originalData.reset(data.getData(), 0, data.getLength());
+    decryptedDataBuffer.reset(encryptedDataBuffer.getData(), 0, 
+        encryptedDataBuffer.getLength());
+    in = new CryptoInputStream(decryptedDataBuffer, 
+        codec, bufferSize, key, iv);
+
+    // Check
+    originalIn = new DataInputStream(new BufferedInputStream(originalData));
+    int expected;
+    do {
+      expected = originalIn.read();
+      assertEquals("Decrypted stream read by byte does not match",
+        expected, in.read());
+    } while (expected != -1);
+
+    LOG.info("SUCCESS! Completed checking " + count + " records");
+    
+    // Check secure random generator
+    testSecureRandom(codec);
+  }
+  
+  /** Test secure random generator */
+  private void testSecureRandom(CryptoCodec codec) {
+    // len = 16
+    checkSecureRandom(codec, 16);
+    // len = 32
+    checkSecureRandom(codec, 32);
+    // len = 128
+    checkSecureRandom(codec, 128);
+  }
+  
+  private void checkSecureRandom(CryptoCodec codec, int len) {
     byte[] rand = new byte[len];
     byte[] rand1 = new byte[len];
     codec.generateSecureRandom(rand);
@@ -56,28 +183,6 @@ public class TestCryptoCodec {
     
     Assert.assertEquals(len, rand.length);
     Assert.assertEquals(len, rand1.length);
-    Assert.assertFalse(bytesArrayEquals(rand, rand1));
+    Assert.assertFalse(Arrays.equals(rand, rand1));
-  }
-  
-  private boolean bytesArrayEquals(byte[] expected, byte[] actual) {
-    if ((expected == null  && actual != null) || 
-        (expected != null && actual == null)) {
-      return false;
-    }
-    if (expected == null && actual == null) {
-      return true;
-    }
-    
-    if (expected.length != actual.length) {
-      return false;
-    }
-    
-    for (int i = 0; i < expected.length; i++) {
-      if (expected[i] != actual[i]) {
-        return false;
-      }
-    }
-    
-    return true;
   }
 }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeCodeLoader.java

@@ -22,6 +22,7 @@ import static org.junit.Assert.*;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.crypto.OpensslCipher;
 import org.apache.hadoop.io.compress.Lz4Codec;
 import org.apache.hadoop.io.compress.SnappyCodec;
 import org.apache.hadoop.io.compress.zlib.ZlibFactory;
@@ -54,6 +55,9 @@ public class TestNativeCodeLoader {
     if (NativeCodeLoader.buildSupportsSnappy()) {
       assertFalse(SnappyCodec.getLibraryName().isEmpty());
     }
+    if (NativeCodeLoader.buildSupportsOpenssl()) {
+      assertFalse(OpensslCipher.getLibraryName().isEmpty());
+    }
     assertFalse(Lz4Codec.getLibraryName().isEmpty());
     LOG.info("TestNativeCodeLoader: libhadoop.so is loaded.");
   }

hadoop-hdfs-project/hadoop-hdfs/CHANGES-fs-encryption.txt

@@ -10,6 +10,9 @@ fs-encryption (Unreleased)
 
   IMPROVEMENTS
 
+    HADOOP-10693. Implementation of AES-CTR CryptoCodec using JNI to OpenSSL
+    (hitliuyi via cmccabe)
+
     HDFS-6387. HDFS CLI admin tool for creating & deleting an
     encryption zone. (clamb)
 

hadoop-project-dist/pom.xml

@@ -41,6 +41,8 @@
     <hadoop.component>UNDEF</hadoop.component>
     <bundle.snappy>false</bundle.snappy>
     <bundle.snappy.in.bin>false</bundle.snappy.in.bin>
+    <bundle.openssl>false</bundle.openssl>
+    <bundle.openssl.in.bin>false</bundle.openssl.in.bin>
   </properties>
   
   <dependencies>
@@ -349,6 +351,10 @@
                           cd "${snappy.lib}"
                           $$TAR *snappy* | (cd $${TARGET_DIR}/; $$UNTAR)
                         fi
+                        if [ "${bundle.openssl}" = "true" ] ; then
+                          cd "${openssl.lib}"
+                          $$TAR *crypto* | (cd $${TARGET_DIR}/; $$UNTAR)
+                        fi
                       fi
                       BIN_DIR="${BUILD_DIR}/bin"
                       if [ -d $${BIN_DIR} ] ; then
@@ -362,6 +368,12 @@
                             $$TAR *snappy* | (cd $${TARGET_BIN_DIR}/; $$UNTAR)
                           fi
                         fi
+                        if [ "${bundle.openssl.in.bin}" = "true" ] ; then
+                          if [ "${bundle.openssl}" = "true" ] ; then
+                            cd "${openssl.lib}"
+                            $$TAR *crypto* | (cd $${TARGET_BIN_DIR}/; $$UNTAR)
+                          fi
+                        fi
                       fi
                     </echo>
                     <exec executable="sh" dir="${project.build.directory}" failonerror="true">

hadoop-project/pom.xml

@@ -1031,6 +1031,7 @@
         <!-- attempt to open a file at this path. -->
         <java.security.egd>file:/dev/urandom</java.security.egd>
         <bundle.snappy.in.bin>true</bundle.snappy.in.bin>
+        <bundle.openssl.in.bin>true</bundle.openssl.in.bin>
       </properties>
       <build>
         <plugins>
@@ -1041,6 +1042,7 @@
               <environmentVariables>
                 <!-- Specify where to look for the native DLL on Windows -->
                 <PATH>${env.PATH};${hadoop.common.build.dir}/bin;${snappy.lib}</PATH>
+                <PATH>${env.PATH};${hadoop.common.build.dir}/bin;${openssl.lib}</PATH>
               </environmentVariables>
             </configuration>
           </plugin>