From 67d1c70c2d70d393abee3a3bd64cd55457f33f9e Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Fri, 2 Oct 2015 17:18:26 +0200 Subject: [PATCH] Remove and forbid use of com.google.common.hash.* This commit removes and now forbids all uses of com.google.common.hash.HashCode, com.google.common.hash.HashFunction, and com.google.common.hash.Hashing across the codebase. This is one of the few remaining steps in the eventual removal of Guava as a dependency. Relates #13224 --- .../common/hash/MessageDigests.java | 80 ++++++++++++++++++ .../http/client/HttpDownloadHelper.java | 13 +-- .../murmur3/Murmur3HashFunctionTests.java | 28 +++---- .../common/hash/MessageDigestsTests.java | 81 +++++++++++++++++++ .../common/hashing/MurmurHash3Tests.java | 45 +++++------ .../plugins/PluginManagerIT.java | 11 +-- .../resources/forbidden/all-signatures.txt | 3 + .../groovy/GroovyScriptEngineService.java | 10 +-- 8 files changed, 213 insertions(+), 58 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/common/hash/MessageDigests.java create mode 100644 core/src/test/java/org/elasticsearch/common/hash/MessageDigestsTests.java diff --git a/core/src/main/java/org/elasticsearch/common/hash/MessageDigests.java b/core/src/main/java/org/elasticsearch/common/hash/MessageDigests.java new file mode 100644 index 00000000000..7dc495f9c3b --- /dev/null +++ b/core/src/main/java/org/elasticsearch/common/hash/MessageDigests.java @@ -0,0 +1,80 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.hash; + +import org.elasticsearch.ElasticsearchException; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +public class MessageDigests { + + private static final MessageDigest MD5_DIGEST; + private static final MessageDigest SHA_1_DIGEST; + private static final MessageDigest SHA_256_DIGEST; + + static { + try { + MD5_DIGEST = MessageDigest.getInstance("MD5"); + SHA_1_DIGEST = MessageDigest.getInstance("SHA-1"); + SHA_256_DIGEST = MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new ElasticsearchException("Unexpected exception creating MessageDigest instance", e); + } + } + + public static MessageDigest md5() { + return cloneAndReset(MD5_DIGEST); + } + + + public static MessageDigest sha1() { + return cloneAndReset(SHA_1_DIGEST); + } + + public static MessageDigest sha256() { + return cloneAndReset(SHA_256_DIGEST); + } + + private static MessageDigest cloneAndReset(MessageDigest messageDigest) { + try { + MessageDigest clone = (MessageDigest) messageDigest.clone(); + clone.reset(); + return clone; + } catch (CloneNotSupportedException e) { + throw new ElasticsearchException("Unexpected exception cloning MessageDigest instance", e); + } + } + + private static final char[] HEX_DIGITS = "0123456789abcdef".toCharArray(); + public static String toHexString(byte[] bytes) { + if (bytes == null) { + throw new NullPointerException("bytes"); + } + StringBuilder sb = new StringBuilder(2 * bytes.length); + + for (int i = 0; i < bytes.length; i++) { + byte b = bytes[i]; + sb.append(HEX_DIGITS[b >> 4 & 0xf]).append(HEX_DIGITS[b & 0xf]); + } + + return sb.toString(); + } +} diff --git a/core/src/main/java/org/elasticsearch/common/http/client/HttpDownloadHelper.java b/core/src/main/java/org/elasticsearch/common/http/client/HttpDownloadHelper.java index ed2de6e5e7d..7fe26ed81d9 100644 --- a/core/src/main/java/org/elasticsearch/common/http/client/HttpDownloadHelper.java +++ b/core/src/main/java/org/elasticsearch/common/http/client/HttpDownloadHelper.java @@ -19,19 +19,22 @@ package org.elasticsearch.common.http.client; -import java.nio.charset.StandardCharsets; -import com.google.common.hash.Hashing; import org.apache.lucene.util.IOUtils; -import org.elasticsearch.*; +import org.elasticsearch.Build; +import org.elasticsearch.ElasticsearchCorruptionException; +import org.elasticsearch.ElasticsearchTimeoutException; +import org.elasticsearch.Version; import org.elasticsearch.common.Base64; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.hash.MessageDigests; import org.elasticsearch.common.unit.TimeValue; import java.io.*; import java.net.HttpURLConnection; import java.net.URL; import java.net.URLConnection; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; @@ -96,7 +99,7 @@ public class HttpDownloadHelper { public static Checksummer SHA1_CHECKSUM = new Checksummer() { @Override public String checksum(byte[] filebytes) { - return Hashing.sha1().hashBytes(filebytes).toString(); + return MessageDigests.toHexString(MessageDigests.sha1().digest(filebytes)); } @Override @@ -109,7 +112,7 @@ public class HttpDownloadHelper { public static Checksummer MD5_CHECKSUM = new Checksummer() { @Override public String checksum(byte[] filebytes) { - return Hashing.md5().hashBytes(filebytes).toString(); + return MessageDigests.toHexString(MessageDigests.md5().digest(filebytes)); } @Override diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/operation/hash/murmur3/Murmur3HashFunctionTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/operation/hash/murmur3/Murmur3HashFunctionTests.java index 23b928db247..ed454aead0d 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/operation/hash/murmur3/Murmur3HashFunctionTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/operation/hash/murmur3/Murmur3HashFunctionTests.java @@ -19,26 +19,24 @@ package org.elasticsearch.cluster.routing.operation.hash.murmur3; -import com.carrotsearch.randomizedtesting.generators.RandomInts; -import com.carrotsearch.randomizedtesting.generators.RandomStrings; -import com.google.common.hash.HashFunction; -import com.google.common.hash.Hashing; import org.elasticsearch.cluster.routing.Murmur3HashFunction; import org.elasticsearch.test.ESTestCase; public class Murmur3HashFunctionTests extends ESTestCase { - public void test() { - // Make sure that we agree with guava - Murmur3HashFunction murmur3 = new Murmur3HashFunction(); - HashFunction guavaMurmur3 = Hashing.murmur3_32(); - for (int i = 0; i < 100; ++i) { - final String id = RandomStrings.randomRealisticUnicodeOfCodepointLength(getRandom(), RandomInts.randomIntBetween(getRandom(), 1, 20)); - //final String id = "0"; - final int hash1 = guavaMurmur3.newHasher().putUnencodedChars(id).hash().asInt(); - final int hash2 = murmur3.hash(id); - assertEquals(hash1, hash2); - } + private static Murmur3HashFunction HASH = new Murmur3HashFunction(); + + public void testKnownValues() { + assertHash(0x5a0cb7c3, "hell"); + assertHash(0xd7c31989, "hello"); + assertHash(0x22ab2984, "hello w"); + assertHash(0xdf0ca123, "hello wo"); + assertHash(0xe7744d61, "hello wor"); + assertHash(0xe07db09c, "The quick brown fox jumps over the lazy dog"); + assertHash(0x4e63d2ad, "The quick brown fox jumps over the lazy cog"); } + private static void assertHash(int expected, String stringInput) { + assertEquals(expected, HASH.hash(stringInput)); + } } diff --git a/core/src/test/java/org/elasticsearch/common/hash/MessageDigestsTests.java b/core/src/test/java/org/elasticsearch/common/hash/MessageDigestsTests.java new file mode 100644 index 00000000000..f8d39fa6bf6 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/common/hash/MessageDigestsTests.java @@ -0,0 +1,81 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.hash; + +import org.elasticsearch.test.ESTestCase; +import org.junit.Test; + +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; + +import static org.junit.Assert.*; + +public class MessageDigestsTests extends ESTestCase { + private void assertHash(String expected, String test, MessageDigest messageDigest) { + String actual = MessageDigests.toHexString(messageDigest.digest(test.getBytes(StandardCharsets.UTF_8))); + assertEquals(expected, actual); + } + + @Test + public void testMd5() throws Exception { + assertHash("d41d8cd98f00b204e9800998ecf8427e", "", MessageDigests.md5()); + assertHash("900150983cd24fb0d6963f7d28e17f72", "abc", MessageDigests.md5()); + assertHash("8215ef0796a20bcaaae116d3876c664a", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", MessageDigests.md5()); + assertHash("7707d6ae4e027c70eea2a935c2296f21", new String(new char[1000000]).replace("\0", "a"), MessageDigests.md5()); + assertHash("9e107d9d372bb6826bd81d3542a419d6", "The quick brown fox jumps over the lazy dog", MessageDigests.md5()); + assertHash("1055d3e698d289f2af8663725127bd4b", "The quick brown fox jumps over the lazy cog", MessageDigests.md5()); + } + + @Test + public void testSha1() throws Exception { + assertHash("da39a3ee5e6b4b0d3255bfef95601890afd80709", "", MessageDigests.sha1()); + assertHash("a9993e364706816aba3e25717850c26c9cd0d89d", "abc", MessageDigests.sha1()); + assertHash("84983e441c3bd26ebaae4aa1f95129e5e54670f1", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", MessageDigests.sha1()); + assertHash("34aa973cd4c4daa4f61eeb2bdbad27316534016f", new String(new char[1000000]).replace("\0", "a"), MessageDigests.sha1()); + assertHash("2fd4e1c67a2d28fced849ee1bb76e7391b93eb12", "The quick brown fox jumps over the lazy dog", MessageDigests.sha1()); + assertHash("de9f2c7fd25e1b3afad3e85a0bd17d9b100db4b3", "The quick brown fox jumps over the lazy cog", MessageDigests.sha1()); + } + + @Test + public void testSha256() throws Exception { + assertHash("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "", MessageDigests.sha256()); + assertHash("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", "abc", MessageDigests.sha256()); + assertHash("248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", MessageDigests.sha256()); + assertHash("cdc76e5c9914fb9281a1c7e284d73e67f1809a48a497200e046d39ccc7112cd0", new String(new char[1000000]).replace("\0", "a"), MessageDigests.sha256()); + assertHash("d7a8fbb307d7809469ca9abcb0082e4f8d5651e46d3cdb762d02d0bf37c9e592", "The quick brown fox jumps over the lazy dog", MessageDigests.sha256()); + assertHash("e4c4d8f3bf76b692de791a173e05321150f7a345b46484fe427f6acc7ecc81be", "The quick brown fox jumps over the lazy cog", MessageDigests.sha256()); + } + + @Test + public void testToHexString() throws Exception { + for (int i = 0; i < 1024; i++) { + BigInteger expected = BigInteger.probablePrime(256, random()); + byte[] bytes = expected.toByteArray(); + String hex = MessageDigests.toHexString(bytes); + String zeros = new String(new char[bytes.length * 2]).replace("\0", "0"); + String expectedAsString = expected.toString(16); + String expectedHex = zeros.substring(expectedAsString.length()) + expectedAsString; + assertEquals(expectedHex, hex); + BigInteger actual = new BigInteger(hex, 16); + assertEquals(expected, actual); + } + } +} diff --git a/core/src/test/java/org/elasticsearch/common/hashing/MurmurHash3Tests.java b/core/src/test/java/org/elasticsearch/common/hashing/MurmurHash3Tests.java index d9d4057f961..cbdfe9ce3aa 100644 --- a/core/src/test/java/org/elasticsearch/common/hashing/MurmurHash3Tests.java +++ b/core/src/test/java/org/elasticsearch/common/hashing/MurmurHash3Tests.java @@ -19,37 +19,34 @@ package org.elasticsearch.common.hashing; -import com.google.common.hash.HashCode; -import com.google.common.hash.Hashing; import org.elasticsearch.common.hash.MurmurHash3; import org.elasticsearch.test.ESTestCase; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.LongBuffer; +import java.io.UnsupportedEncodingException; +import java.nio.charset.StandardCharsets; public class MurmurHash3Tests extends ESTestCase { - - public void testHash128() { - final int iters = scaledRandomIntBetween(100, 5000); - for (int i = 0; i < iters; ++i) { - final int seed = randomInt(); - final int offset = randomInt(20); - final int len = randomInt(randomBoolean() ? 20 : 200); - final byte[] bytes = new byte[len + offset + randomInt(3)]; - getRandom().nextBytes(bytes); - HashCode h1 = Hashing.murmur3_128(seed).hashBytes(bytes, offset, len); - MurmurHash3.Hash128 h2 = MurmurHash3.hash128(bytes, offset, len, seed, new MurmurHash3.Hash128()); - assertEquals(h1, h2); - } + public void testKnownValues() throws UnsupportedEncodingException { + assertHash(0x629942693e10f867L, 0x92db0b82baeb5347L, "hell", 0); + assertHash(0xa78ddff5adae8d10L, 0x128900ef20900135L, "hello", 1); + assertHash(0x8a486b23f422e826L, 0xf962a2c58947765fL, "hello ", 2); + assertHash(0x2ea59f466f6bed8cL, 0xc610990acc428a17L, "hello w", 3); + assertHash(0x79f6305a386c572cL, 0x46305aed3483b94eL, "hello wo", 4); + assertHash(0xc2219d213ec1f1b5L, 0xa1d8e2e0a52785bdL, "hello wor", 5); + assertHash(0xe34bbc7bbc071b6cL, 0x7a433ca9c49a9347L, "The quick brown fox jumps over the lazy dog", 0); + assertHash(0x658ca970ff85269aL, 0x43fee3eaa68e5c3eL, "The quick brown fox jumps over the lazy cog", 0); } - private void assertEquals(HashCode h1, MurmurHash3.Hash128 h2) { - final LongBuffer longs = ByteBuffer.wrap(h1.asBytes()).order(ByteOrder.LITTLE_ENDIAN).asLongBuffer(); - assertEquals(2, longs.limit()); - assertEquals(h1.asLong(), h2.h1); - assertEquals(longs.get(), h2.h1); - assertEquals(longs.get(), h2.h2); + private static void assertHash(long lower, long upper, String inputString, long seed) { + byte[] bytes = inputString.getBytes(StandardCharsets.UTF_8); + MurmurHash3.Hash128 expected = new MurmurHash3.Hash128(); + expected.h1 = lower; + expected.h2 = upper; + assertHash(expected, MurmurHash3.hash128(bytes, 0, bytes.length, seed, new MurmurHash3.Hash128())); } + private static void assertHash(MurmurHash3.Hash128 expected, MurmurHash3.Hash128 actual) { + assertEquals(expected.h1, actual.h1); + assertEquals(expected.h2, actual.h2); + } } diff --git a/core/src/test/java/org/elasticsearch/plugins/PluginManagerIT.java b/core/src/test/java/org/elasticsearch/plugins/PluginManagerIT.java index c76bef55abe..1ffce8d71c2 100644 --- a/core/src/test/java/org/elasticsearch/plugins/PluginManagerIT.java +++ b/core/src/test/java/org/elasticsearch/plugins/PluginManagerIT.java @@ -18,9 +18,6 @@ */ package org.elasticsearch.plugins; -import java.nio.charset.StandardCharsets; -import com.google.common.hash.Hashing; - import org.apache.http.impl.client.HttpClients; import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.Version; @@ -28,6 +25,7 @@ import org.elasticsearch.common.Base64; import org.elasticsearch.common.cli.CliTool; import org.elasticsearch.common.cli.CliTool.ExitStatus; import org.elasticsearch.common.cli.CliToolTestCase.CaptureOutputTerminal; +import org.elasticsearch.common.hash.MessageDigests; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.node.internal.InternalSettingsPreparer; @@ -46,16 +44,15 @@ import org.jboss.netty.handler.ssl.util.InsecureTrustManagerFactory; import org.jboss.netty.handler.ssl.util.SelfSignedCertificate; import org.junit.After; import org.junit.Before; -import org.junit.Test; import javax.net.ssl.HttpsURLConnection; import javax.net.ssl.SSLContext; import javax.net.ssl.SSLSocketFactory; - import java.io.BufferedWriter; import java.io.IOException; import java.net.InetAddress; import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; @@ -109,7 +106,7 @@ public class PluginManagerIT extends ESIntegTestCase { } private void writeSha1(Path file, boolean corrupt) throws IOException { - String sha1Hex = Hashing.sha1().hashBytes(Files.readAllBytes(file)).toString(); + String sha1Hex = MessageDigests.toHexString(MessageDigests.sha1().digest(Files.readAllBytes(file))); try (BufferedWriter out = Files.newBufferedWriter(file.resolveSibling(file.getFileName() + ".sha1"), StandardCharsets.UTF_8)) { out.write(sha1Hex); if (corrupt) { @@ -119,7 +116,7 @@ public class PluginManagerIT extends ESIntegTestCase { } private void writeMd5(Path file, boolean corrupt) throws IOException { - String md5Hex = Hashing.md5().hashBytes(Files.readAllBytes(file)).toString(); + String md5Hex = MessageDigests.toHexString(MessageDigests.md5().digest(Files.readAllBytes(file))); try (BufferedWriter out = Files.newBufferedWriter(file.resolveSibling(file.getFileName() + ".md5"), StandardCharsets.UTF_8)) { out.write(md5Hex); if (corrupt) { diff --git a/dev-tools/src/main/resources/forbidden/all-signatures.txt b/dev-tools/src/main/resources/forbidden/all-signatures.txt index 68c6ea7317b..e57a86227cd 100644 --- a/dev-tools/src/main/resources/forbidden/all-signatures.txt +++ b/dev-tools/src/main/resources/forbidden/all-signatures.txt @@ -130,6 +130,9 @@ com.google.common.primitives.Ints com.google.common.collect.ImmutableSet com.google.common.collect.ImmutableSet$Builder com.google.common.io.Resources +com.google.common.hash.HashCode +com.google.common.hash.HashFunction +com.google.common.hash.Hashing @defaultMessage Do not violate java's access system java.lang.reflect.AccessibleObject#setAccessible(boolean) diff --git a/plugins/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java b/plugins/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java index 935916149fb..cd478bef604 100644 --- a/plugins/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java +++ b/plugins/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java @@ -19,14 +19,9 @@ package org.elasticsearch.script.groovy; -import java.nio.charset.StandardCharsets; - -import com.google.common.hash.Hashing; - import groovy.lang.Binding; import groovy.lang.GroovyClassLoader; import groovy.lang.Script; - import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.Scorer; import org.codehaus.groovy.ast.ClassCodeExpressionTransformer; @@ -41,9 +36,9 @@ import org.codehaus.groovy.control.SourceUnit; import org.codehaus.groovy.control.customizers.CompilationCustomizer; import org.codehaus.groovy.control.customizers.ImportCustomizer; import org.elasticsearch.SpecialPermission; -import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.hash.MessageDigests; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.settings.Settings; @@ -53,6 +48,7 @@ import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; import java.security.AccessController; import java.security.PrivilegedAction; import java.util.HashMap; @@ -172,7 +168,7 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri if (sm != null) { sm.checkPermission(new SpecialPermission()); } - return loader.parseClass(script, Hashing.sha1().hashString(script, StandardCharsets.UTF_8).toString()); + return loader.parseClass(script, MessageDigests.toHexString(MessageDigests.sha1().digest(script.getBytes(StandardCharsets.UTF_8)))); } catch (Throwable e) { if (logger.isTraceEnabled()) { logger.trace("exception compiling Groovy script:", e);