honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #13907 from jasontedor/hash-be-gone 

Remove and forbid use of com.google.common.hash.*
This commit is contained in:
Jason Tedor 2015-10-04 16:26:12 -04:00
parent 96206dfd2a 67d1c70c2d
commit e5a10e9520
8 changed files with 213 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
core/src/main/java/org/elasticsearch/common/hash/MessageDigests.java Normal file
View File

@ -0,0 +1,80 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.common.hash;
import org.elasticsearch.ElasticsearchException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
public class MessageDigests {
private static final MessageDigest MD5_DIGEST;
private static final MessageDigest SHA_1_DIGEST;
private static final MessageDigest SHA_256_DIGEST;
static {
try {
MD5_DIGEST = MessageDigest.getInstance("MD5");
SHA_1_DIGEST = MessageDigest.getInstance("SHA-1");
SHA_256_DIGEST = MessageDigest.getInstance("SHA-256");
} catch (NoSuchAlgorithmException e) {
throw new ElasticsearchException("Unexpected exception creating MessageDigest instance", e);
}
}
public static MessageDigest md5() {
return cloneAndReset(MD5_DIGEST);
}
public static MessageDigest sha1() {
return cloneAndReset(SHA_1_DIGEST);
}
public static MessageDigest sha256() {
return cloneAndReset(SHA_256_DIGEST);
}
private static MessageDigest cloneAndReset(MessageDigest messageDigest) {
try {
MessageDigest clone = (MessageDigest) messageDigest.clone();
clone.reset();
return clone;
} catch (CloneNotSupportedException e) {
throw new ElasticsearchException("Unexpected exception cloning MessageDigest instance", e);
}
}
private static final char[] HEX_DIGITS = "0123456789abcdef".toCharArray();
public static String toHexString(byte[] bytes) {
if (bytes == null) {
throw new NullPointerException("bytes");
}
StringBuilder sb = new StringBuilder(2 * bytes.length);
for (int i = 0; i < bytes.length; i++) {
byte b = bytes[i];
sb.append(HEX_DIGITS[b >> 4 & 0xf]).append(HEX_DIGITS[b & 0xf]);
}
return sb.toString();
}
}

13
core/src/main/java/org/elasticsearch/common/http/client/HttpDownloadHelper.java
View File

@ -19,19 +19,22 @@
package org.elasticsearch.common.http.client;
import java.nio.charset.StandardCharsets;
import com.google.common.hash.Hashing;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.*;
import org.elasticsearch.Build;
import org.elasticsearch.ElasticsearchCorruptionException;
import org.elasticsearch.ElasticsearchTimeoutException;
import org.elasticsearch.Version;
import org.elasticsearch.common.Base64;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.hash.MessageDigests;
import org.elasticsearch.common.unit.TimeValue;
import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
@ -96,7 +99,7 @@ public class HttpDownloadHelper {
public static Checksummer SHA1_CHECKSUM = new Checksummer() {
@Override
public String checksum(byte[] filebytes) {
return Hashing.sha1().hashBytes(filebytes).toString();
return MessageDigests.toHexString(MessageDigests.sha1().digest(filebytes));
}
@Override
@ -109,7 +112,7 @@ public class HttpDownloadHelper {
public static Checksummer MD5_CHECKSUM = new Checksummer() {
@Override
public String checksum(byte[] filebytes) {
return Hashing.md5().hashBytes(filebytes).toString();
return MessageDigests.toHexString(MessageDigests.md5().digest(filebytes));
}
@Override

28
core/src/test/java/org/elasticsearch/cluster/routing/operation/hash/murmur3/Murmur3HashFunctionTests.java
View File

@ -19,26 +19,24 @@
package org.elasticsearch.cluster.routing.operation.hash.murmur3;
import com.carrotsearch.randomizedtesting.generators.RandomInts;
import com.carrotsearch.randomizedtesting.generators.RandomStrings;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
import org.elasticsearch.cluster.routing.Murmur3HashFunction;
import org.elasticsearch.test.ESTestCase;
public class Murmur3HashFunctionTests extends ESTestCase {
public void test() {
// Make sure that we agree with guava
Murmur3HashFunction murmur3 = new Murmur3HashFunction();
HashFunction guavaMurmur3 = Hashing.murmur3_32();
for (int i = 0; i < 100; ++i) {
final String id = RandomStrings.randomRealisticUnicodeOfCodepointLength(getRandom(), RandomInts.randomIntBetween(getRandom(), 1, 20));
//final String id = "0";
final int hash1 = guavaMurmur3.newHasher().putUnencodedChars(id).hash().asInt();
final int hash2 = murmur3.hash(id);
assertEquals(hash1, hash2);
}
private static Murmur3HashFunction HASH = new Murmur3HashFunction();
public void testKnownValues() {
assertHash(0x5a0cb7c3, "hell");
assertHash(0xd7c31989, "hello");
assertHash(0x22ab2984, "hello w");
assertHash(0xdf0ca123, "hello wo");
assertHash(0xe7744d61, "hello wor");
assertHash(0xe07db09c, "The quick brown fox jumps over the lazy dog");
assertHash(0x4e63d2ad, "The quick brown fox jumps over the lazy cog");
}
private static void assertHash(int expected, String stringInput) {
assertEquals(expected, HASH.hash(stringInput));
}
}

81
core/src/test/java/org/elasticsearch/common/hash/MessageDigestsTests.java Normal file
View File

@ -0,0 +1,81 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.common.hash;
import org.elasticsearch.test.ESTestCase;
import org.junit.Test;
import java.math.BigInteger;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import static org.junit.Assert.*;
public class MessageDigestsTests extends ESTestCase {
private void assertHash(String expected, String test, MessageDigest messageDigest) {
String actual = MessageDigests.toHexString(messageDigest.digest(test.getBytes(StandardCharsets.UTF_8)));
assertEquals(expected, actual);
}
@Test
public void testMd5() throws Exception {
assertHash("d41d8cd98f00b204e9800998ecf8427e", "", MessageDigests.md5());
assertHash("900150983cd24fb0d6963f7d28e17f72", "abc", MessageDigests.md5());
assertHash("8215ef0796a20bcaaae116d3876c664a", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", MessageDigests.md5());
assertHash("7707d6ae4e027c70eea2a935c2296f21", new String(new char[1000000]).replace("\0", "a"), MessageDigests.md5());
assertHash("9e107d9d372bb6826bd81d3542a419d6", "The quick brown fox jumps over the lazy dog", MessageDigests.md5());
assertHash("1055d3e698d289f2af8663725127bd4b", "The quick brown fox jumps over the lazy cog", MessageDigests.md5());
}
@Test
public void testSha1() throws Exception {
assertHash("da39a3ee5e6b4b0d3255bfef95601890afd80709", "", MessageDigests.sha1());
assertHash("a9993e364706816aba3e25717850c26c9cd0d89d", "abc", MessageDigests.sha1());
assertHash("84983e441c3bd26ebaae4aa1f95129e5e54670f1", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", MessageDigests.sha1());
assertHash("34aa973cd4c4daa4f61eeb2bdbad27316534016f", new String(new char[1000000]).replace("\0", "a"), MessageDigests.sha1());
assertHash("2fd4e1c67a2d28fced849ee1bb76e7391b93eb12", "The quick brown fox jumps over the lazy dog", MessageDigests.sha1());
assertHash("de9f2c7fd25e1b3afad3e85a0bd17d9b100db4b3", "The quick brown fox jumps over the lazy cog", MessageDigests.sha1());
}
@Test
public void testSha256() throws Exception {
assertHash("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "", MessageDigests.sha256());
assertHash("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", "abc", MessageDigests.sha256());
assertHash("248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", MessageDigests.sha256());
assertHash("cdc76e5c9914fb9281a1c7e284d73e67f1809a48a497200e046d39ccc7112cd0", new String(new char[1000000]).replace("\0", "a"), MessageDigests.sha256());
assertHash("d7a8fbb307d7809469ca9abcb0082e4f8d5651e46d3cdb762d02d0bf37c9e592", "The quick brown fox jumps over the lazy dog", MessageDigests.sha256());
assertHash("e4c4d8f3bf76b692de791a173e05321150f7a345b46484fe427f6acc7ecc81be", "The quick brown fox jumps over the lazy cog", MessageDigests.sha256());
}
@Test
public void testToHexString() throws Exception {
for (int i = 0; i < 1024; i++) {
BigInteger expected = BigInteger.probablePrime(256, random());
byte[] bytes = expected.toByteArray();
String hex = MessageDigests.toHexString(bytes);
String zeros = new String(new char[bytes.length * 2]).replace("\0", "0");
String expectedAsString = expected.toString(16);
String expectedHex = zeros.substring(expectedAsString.length()) + expectedAsString;
assertEquals(expectedHex, hex);
BigInteger actual = new BigInteger(hex, 16);
assertEquals(expected, actual);
}
}
}

45
core/src/test/java/org/elasticsearch/common/hashing/MurmurHash3Tests.java
View File

@ -19,37 +19,34 @@
package org.elasticsearch.common.hashing;
import com.google.common.hash.HashCode;
import com.google.common.hash.Hashing;
import org.elasticsearch.common.hash.MurmurHash3;
import org.elasticsearch.test.ESTestCase;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.LongBuffer;
import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets;
public class MurmurHash3Tests extends ESTestCase {
public void testHash128() {
final int iters = scaledRandomIntBetween(100, 5000);
for (int i = 0; i < iters; ++i) {
final int seed = randomInt();
final int offset = randomInt(20);
final int len = randomInt(randomBoolean() ? 20 : 200);
final byte[] bytes = new byte[len + offset + randomInt(3)];
getRandom().nextBytes(bytes);
HashCode h1 = Hashing.murmur3_128(seed).hashBytes(bytes, offset, len);
MurmurHash3.Hash128 h2 = MurmurHash3.hash128(bytes, offset, len, seed, new MurmurHash3.Hash128());
assertEquals(h1, h2);
}
public void testKnownValues() throws UnsupportedEncodingException {
assertHash(0x629942693e10f867L, 0x92db0b82baeb5347L, "hell", 0);
assertHash(0xa78ddff5adae8d10L, 0x128900ef20900135L, "hello", 1);
assertHash(0x8a486b23f422e826L, 0xf962a2c58947765fL, "hello ", 2);
assertHash(0x2ea59f466f6bed8cL, 0xc610990acc428a17L, "hello w", 3);
assertHash(0x79f6305a386c572cL, 0x46305aed3483b94eL, "hello wo", 4);
assertHash(0xc2219d213ec1f1b5L, 0xa1d8e2e0a52785bdL, "hello wor", 5);
assertHash(0xe34bbc7bbc071b6cL, 0x7a433ca9c49a9347L, "The quick brown fox jumps over the lazy dog", 0);
assertHash(0x658ca970ff85269aL, 0x43fee3eaa68e5c3eL, "The quick brown fox jumps over the lazy cog", 0);
}
private void assertEquals(HashCode h1, MurmurHash3.Hash128 h2) {
final LongBuffer longs = ByteBuffer.wrap(h1.asBytes()).order(ByteOrder.LITTLE_ENDIAN).asLongBuffer();
assertEquals(2, longs.limit());
assertEquals(h1.asLong(), h2.h1);
assertEquals(longs.get(), h2.h1);
assertEquals(longs.get(), h2.h2);
private static void assertHash(long lower, long upper, String inputString, long seed) {
byte[] bytes = inputString.getBytes(StandardCharsets.UTF_8);
MurmurHash3.Hash128 expected = new MurmurHash3.Hash128();
expected.h1 = lower;
expected.h2 = upper;
assertHash(expected, MurmurHash3.hash128(bytes, 0, bytes.length, seed, new MurmurHash3.Hash128()));
}
private static void assertHash(MurmurHash3.Hash128 expected, MurmurHash3.Hash128 actual) {
assertEquals(expected.h1, actual.h1);
assertEquals(expected.h2, actual.h2);
}
}

11
core/src/test/java/org/elasticsearch/plugins/PluginManagerIT.java
View File

@ -18,9 +18,6 @@
*/
package org.elasticsearch.plugins;
import java.nio.charset.StandardCharsets;
import com.google.common.hash.Hashing;
import org.apache.http.impl.client.HttpClients;
import org.apache.lucene.util.LuceneTestCase;
import org.elasticsearch.Version;
@ -28,6 +25,7 @@ import org.elasticsearch.common.Base64;
import org.elasticsearch.common.cli.CliTool;
import org.elasticsearch.common.cli.CliTool.ExitStatus;
import org.elasticsearch.common.cli.CliToolTestCase.CaptureOutputTerminal;
import org.elasticsearch.common.hash.MessageDigests;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.node.internal.InternalSettingsPreparer;
@ -46,16 +44,15 @@ import org.jboss.netty.handler.ssl.util.InsecureTrustManagerFactory;
import org.jboss.netty.handler.ssl.util.SelfSignedCertificate;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSocketFactory;
import java.io.BufferedWriter;
import java.io.IOException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
@ -109,7 +106,7 @@ public class PluginManagerIT extends ESIntegTestCase {
}
private void writeSha1(Path file, boolean corrupt) throws IOException {
String sha1Hex = Hashing.sha1().hashBytes(Files.readAllBytes(file)).toString();
String sha1Hex = MessageDigests.toHexString(MessageDigests.sha1().digest(Files.readAllBytes(file)));
try (BufferedWriter out = Files.newBufferedWriter(file.resolveSibling(file.getFileName() + ".sha1"), StandardCharsets.UTF_8)) {
out.write(sha1Hex);
if (corrupt) {
@ -119,7 +116,7 @@ public class PluginManagerIT extends ESIntegTestCase {
}
private void writeMd5(Path file, boolean corrupt) throws IOException {
String md5Hex = Hashing.md5().hashBytes(Files.readAllBytes(file)).toString();
String md5Hex = MessageDigests.toHexString(MessageDigests.md5().digest(Files.readAllBytes(file)));
try (BufferedWriter out = Files.newBufferedWriter(file.resolveSibling(file.getFileName() + ".md5"), StandardCharsets.UTF_8)) {
out.write(md5Hex);
if (corrupt) {

3
dev-tools/src/main/resources/forbidden/all-signatures.txt
View File

@ -130,6 +130,9 @@ com.google.common.primitives.Ints
com.google.common.collect.ImmutableSet
com.google.common.collect.ImmutableSet$Builder
com.google.common.io.Resources
com.google.common.hash.HashCode
com.google.common.hash.HashFunction
com.google.common.hash.Hashing
@defaultMessage Do not violate java's access system
java.lang.reflect.AccessibleObject#setAccessible(boolean)

10
plugins/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java
View File

@ -19,14 +19,9 @@
package org.elasticsearch.script.groovy;
import java.nio.charset.StandardCharsets;
import com.google.common.hash.Hashing;
import groovy.lang.Binding;
import groovy.lang.GroovyClassLoader;
import groovy.lang.Script;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.Scorer;
import org.codehaus.groovy.ast.ClassCodeExpressionTransformer;
@ -41,9 +36,9 @@ import org.codehaus.groovy.control.SourceUnit;
import org.codehaus.groovy.control.customizers.CompilationCustomizer;
import org.codehaus.groovy.control.customizers.ImportCustomizer;
import org.elasticsearch.SpecialPermission;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.hash.MessageDigests;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.settings.Settings;
@ -53,6 +48,7 @@ import org.elasticsearch.search.lookup.SearchLookup;
import java.io.IOException;
import java.math.BigDecimal;
import java.nio.charset.StandardCharsets;
import java.security.AccessController;
import java.security.PrivilegedAction;
import java.util.HashMap;
@ -172,7 +168,7 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri
if (sm != null) {
sm.checkPermission(new SpecialPermission());
}
return loader.parseClass(script, Hashing.sha1().hashString(script, StandardCharsets.UTF_8).toString());
return loader.parseClass(script, MessageDigests.toHexString(MessageDigests.sha1().digest(script.getBytes(StandardCharsets.UTF_8))));
} catch (Throwable e) {
if (logger.isTraceEnabled()) {
logger.trace("exception compiling Groovy script:", e);