From 6fac3e9b611c43b4f7a97c80f86dd761782cef09 Mon Sep 17 00:00:00 2001 From: Colin McCabe Date: Tue, 22 Jul 2014 00:58:10 +0000 Subject: [PATCH] HADOOP-10870. Failed to load OpenSSL cipher error logs on systems with old openssl versions (cmccabe) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/fs-encryption@1612440 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES-fs-encryption.txt | 3 +++ .../crypto/OpensslAesCtrCryptoCodec.java | 5 ++-- .../apache/hadoop/crypto/OpensslCipher.java | 23 +++++++++++-------- .../hadoop/util/NativeLibraryChecker.java | 16 +++++++------ .../apache/hadoop/crypto/TestCryptoCodec.java | 16 ++++++------- .../hadoop/crypto/TestOpensslCipher.java | 13 ++++------- 6 files changed, 40 insertions(+), 36 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES-fs-encryption.txt b/hadoop-common-project/hadoop-common/CHANGES-fs-encryption.txt index 4c365bf0aeb..3f9f13df0e0 100644 --- a/hadoop-common-project/hadoop-common/CHANGES-fs-encryption.txt +++ b/hadoop-common-project/hadoop-common/CHANGES-fs-encryption.txt @@ -40,6 +40,9 @@ fs-encryption (Unreleased) HADOOP-10735. Fall back AesCtrCryptoCodec implementation from OpenSSL to JCE if non native support. (Yi Liu) + HADOOP-10870. Failed to load OpenSSL cipher error logs on systems with old + openssl versions (cmccabe) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslAesCtrCryptoCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslAesCtrCryptoCodec.java index 7db7b063e5c..4ca79b307d6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslAesCtrCryptoCodec.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslAesCtrCryptoCodec.java @@ -47,8 +47,9 @@ public class OpensslAesCtrCryptoCodec extends AesCtrCryptoCodec { private Random random; public OpensslAesCtrCryptoCodec() { - if (!OpensslCipher.isNativeCodeLoaded()) { - throw new RuntimeException("Failed to load OpenSSL Cipher."); + String loadingFailureReason = OpensslCipher.getLoadingFailureReason(); + if (loadingFailureReason != null) { + throw new RuntimeException(loadingFailureReason); } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java index 652a8b4c324..264652b202a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java @@ -76,21 +76,26 @@ public final class OpensslCipher { private final int alg; private final int padding; - private static boolean nativeCipherLoaded = false; + private static final String loadingFailureReason; + static { - if (NativeCodeLoader.isNativeCodeLoaded() && - NativeCodeLoader.buildSupportsOpenssl()) { - try { + String loadingFailure = null; + try { + if (!NativeCodeLoader.buildSupportsOpenssl()) { + loadingFailure = "build does not support openssl."; + } else { initIDs(); - nativeCipherLoaded = true; - } catch (Throwable t) { - LOG.error("Failed to load OpenSSL Cipher.", t); } + } catch (Throwable t) { + loadingFailure = t.getMessage(); + LOG.debug("Failed to load OpenSSL Cipher.", t); + } finally { + loadingFailureReason = loadingFailure; } } - public static boolean isNativeCodeLoaded() { - return nativeCipherLoaded; + public static String getLoadingFailureReason() { + return loadingFailureReason; } private OpensslCipher(long context, int alg, int padding) { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java index 4891f03cbb8..0d87bceda17 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java @@ -58,14 +58,14 @@ public class NativeLibraryChecker { boolean nativeHadoopLoaded = NativeCodeLoader.isNativeCodeLoaded(); boolean zlibLoaded = false; boolean snappyLoaded = false; - boolean opensslLoaded = false; // lz4 is linked within libhadoop boolean lz4Loaded = nativeHadoopLoaded; boolean bzip2Loaded = Bzip2Factory.isNativeBzip2Loaded(conf); + boolean openSslLoaded = false; + String openSslDetail = ""; String hadoopLibraryName = ""; String zlibLibraryName = ""; String snappyLibraryName = ""; - String opensslLibraryName = ""; String lz4LibraryName = ""; String bzip2LibraryName = ""; if (nativeHadoopLoaded) { @@ -79,10 +79,12 @@ public class NativeLibraryChecker { if (snappyLoaded && NativeCodeLoader.buildSupportsSnappy()) { snappyLibraryName = SnappyCodec.getLibraryName(); } - opensslLoaded = NativeCodeLoader.buildSupportsOpenssl() && - OpensslCipher.isNativeCodeLoaded(); - if (opensslLoaded) { - opensslLibraryName = OpensslCipher.getLibraryName(); + if (OpensslCipher.getLoadingFailureReason() != null) { + openSslDetail = OpensslCipher.getLoadingFailureReason(); + openSslLoaded = false; + } else { + openSslDetail = OpensslCipher.getLibraryName(); + openSslLoaded = true; } if (lz4Loaded) { lz4LibraryName = Lz4Codec.getLibraryName(); @@ -97,7 +99,7 @@ public class NativeLibraryChecker { System.out.printf("snappy: %b %s\n", snappyLoaded, snappyLibraryName); System.out.printf("lz4: %b %s\n", lz4Loaded, lz4LibraryName); System.out.printf("bzip2: %b %s\n", bzip2Loaded, bzip2LibraryName); - System.out.printf("openssl: %b %s\n", opensslLoaded, opensslLibraryName); + System.out.printf("openssl: %b %s\n", openSslLoaded, openSslDetail); if ((!nativeHadoopLoaded) || (checkAll && !(zlibLoaded && snappyLoaded && lz4Loaded && bzip2Loaded))) { // return 1 to indicated check failed diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java index d95052815cf..49b5056a86f 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java @@ -38,6 +38,7 @@ import org.apache.hadoop.io.RandomDatum; import org.apache.hadoop.util.NativeCodeLoader; import org.apache.hadoop.util.ReflectionUtils; import org.junit.Assert; +import org.junit.Assume; import org.junit.Test; public class TestCryptoCodec { @@ -62,15 +63,12 @@ public class TestCryptoCodec { @Test(timeout=1200000) public void testOpensslAesCtrCryptoCodec() throws Exception { - if (NativeCodeLoader.buildSupportsOpenssl()) { - Assert.assertTrue(OpensslCipher.isNativeCodeLoaded()); - } - if (OpensslCipher.isNativeCodeLoaded()) { - cryptoCodecTest(conf, seed, 0, - "org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec"); - cryptoCodecTest(conf, seed, count, - "org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec"); - } + Assume.assumeTrue(NativeCodeLoader.buildSupportsOpenssl()); + Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason()); + cryptoCodecTest(conf, seed, 0, + "org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec"); + cryptoCodecTest(conf, seed, count, + "org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec"); } private void cryptoCodecTest(Configuration conf, int seed, int count, diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestOpensslCipher.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestOpensslCipher.java index b3a894a164f..966a88723a2 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestOpensslCipher.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestOpensslCipher.java @@ -24,6 +24,7 @@ import javax.crypto.NoSuchPaddingException; import javax.crypto.ShortBufferException; import org.apache.hadoop.test.GenericTestUtils; +import org.junit.Assume; import org.junit.Assert; import org.junit.Test; @@ -35,9 +36,7 @@ public class TestOpensslCipher { @Test(timeout=120000) public void testGetInstance() throws Exception { - if (!OpensslCipher.isNativeCodeLoaded()) { - return; - } + Assume.assumeTrue(OpensslCipher.getLoadingFailureReason() == null); OpensslCipher cipher = OpensslCipher.getInstance("AES/CTR/NoPadding"); Assert.assertTrue(cipher != null); @@ -58,9 +57,7 @@ public class TestOpensslCipher { @Test(timeout=120000) public void testUpdateArguments() throws Exception { - if (!OpensslCipher.isNativeCodeLoaded()) { - return; - } + Assume.assumeTrue(OpensslCipher.getLoadingFailureReason() == null); OpensslCipher cipher = OpensslCipher.getInstance("AES/CTR/NoPadding"); Assert.assertTrue(cipher != null); @@ -93,9 +90,7 @@ public class TestOpensslCipher { @Test(timeout=120000) public void testDoFinalArguments() throws Exception { - if (!OpensslCipher.isNativeCodeLoaded()) { - return; - } + Assume.assumeTrue(OpensslCipher.getLoadingFailureReason() == null); OpensslCipher cipher = OpensslCipher.getInstance("AES/CTR/NoPadding"); Assert.assertTrue(cipher != null);