Core: Drop UnsafeUtils.
This class potentially does unaligned memory access and does not bring much now that we switched to global ords for terms aggregations. Close #6962
This commit is contained in:
Adrien Grand
parent
f39d4e1f89
commit
08f8731b6f
|
|
@ -20,7 +20,6 @@ package org.elasticsearch.common.bytes;
|
|||
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.elasticsearch.common.io.stream.StreamInput;
|
||||
import org.elasticsearch.common.util.UnsafeUtils;
|
||||
import org.jboss.netty.buffer.ChannelBuffer;
|
||||
|
||||
import java.io.IOException;
|
||||
|
|
@ -42,16 +41,11 @@ public interface BytesReference {
|
|||
return false;
|
||||
}
|
||||
|
||||
if (a.hasArray() && b.hasArray()) {
|
||||
// court-circuit to compare several bytes at once
|
||||
return UnsafeUtils.equals(a.array(), a.arrayOffset(), b.array(), b.arrayOffset(), a.length());
|
||||
} else {
|
||||
return slowBytesEquals(a, b);
|
||||
}
|
||||
return bytesEquals(a, b);
|
||||
}
|
||||
|
||||
// pkg-private for testing
|
||||
static boolean slowBytesEquals(BytesReference a, BytesReference b) {
|
||||
static boolean bytesEquals(BytesReference a, BytesReference b) {
|
||||
assert a.length() == b.length();
|
||||
for (int i = 0, end = a.length(); i < end; ++i) {
|
||||
if (a.get(i) != b.get(i)) {
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@
|
|||
|
||||
package org.elasticsearch.common.hash;
|
||||
|
||||
import org.elasticsearch.common.util.UnsafeUtils;
|
||||
import org.elasticsearch.common.util.ByteUtils;
|
||||
|
||||
|
||||
/**
|
||||
|
|
@ -41,7 +41,7 @@ public enum MurmurHash3 {
|
|||
protected static long getblock(byte[] key, int offset, int index) {
|
||||
int i_8 = index << 3;
|
||||
int blockOffset = offset + i_8;
|
||||
return UnsafeUtils.readLongLE(key, blockOffset);
|
||||
return ByteUtils.readLongLE(key, blockOffset);
|
||||
}
|
||||
|
||||
protected static long fmix(long k) {
|
||||
|
|
@ -68,8 +68,8 @@ public enum MurmurHash3 {
|
|||
final int len16 = length & 0xFFFFFFF0; // higher multiple of 16 that is lower than or equal to length
|
||||
final int end = offset + len16;
|
||||
for (int i = offset; i < end; i += 16) {
|
||||
long k1 = UnsafeUtils.readLongLE(key, i);
|
||||
long k2 = UnsafeUtils.readLongLE(key, i + 8);
|
||||
long k1 = ByteUtils.readLongLE(key, i);
|
||||
long k2 = ByteUtils.readLongLE(key, i + 8);
|
||||
|
||||
k1 *= C1;
|
||||
k1 = Long.rotateLeft(k1, 31);
|
||||
|
|
|
|||
|
|
@ -77,7 +77,7 @@ public final class BytesRefHash extends AbstractHash {
|
|||
final long slot = slot(rehash(code), mask);
|
||||
for (long index = slot; ; index = nextSlot(index, mask)) {
|
||||
final long id = id(index);
|
||||
if (id == -1L || UnsafeUtils.equals(key, get(id, spare))) {
|
||||
if (id == -1L || key.bytesEquals(get(id, spare))) {
|
||||
return id;
|
||||
}
|
||||
}
|
||||
|
|
@ -99,7 +99,7 @@ public final class BytesRefHash extends AbstractHash {
|
|||
append(id, key, code);
|
||||
++size;
|
||||
return id;
|
||||
} else if (UnsafeUtils.equals(key, get(curId, spare))) {
|
||||
} else if (key.bytesEquals(get(curId, spare))) {
|
||||
return -1 - curId;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,145 +0,0 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.common.util;
|
||||
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import sun.misc.Unsafe;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
import java.nio.ByteOrder;
|
||||
|
||||
/** Utility methods that use {@link Unsafe}. */
|
||||
public enum UnsafeUtils {
|
||||
;
|
||||
|
||||
private static final Unsafe UNSAFE;
|
||||
private static final long BYTE_ARRAY_OFFSET;
|
||||
private static final int BYTE_ARRAY_SCALE;
|
||||
|
||||
static {
|
||||
try {
|
||||
Field theUnsafe = Unsafe.class.getDeclaredField("theUnsafe");
|
||||
theUnsafe.setAccessible(true);
|
||||
UNSAFE = (Unsafe) theUnsafe.get(null);
|
||||
BYTE_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(byte[].class);
|
||||
BYTE_ARRAY_SCALE = UNSAFE.arrayIndexScale(byte[].class);
|
||||
} catch (IllegalAccessException e) {
|
||||
throw new ExceptionInInitializerError("Cannot access Unsafe");
|
||||
} catch (NoSuchFieldException e) {
|
||||
throw new ExceptionInInitializerError("Cannot access Unsafe");
|
||||
} catch (SecurityException e) {
|
||||
throw new ExceptionInInitializerError("Cannot access Unsafe");
|
||||
}
|
||||
}
|
||||
|
||||
// Don't expose these methods directly, they are too easy to mis-use since they depend on the byte order.
|
||||
// If you need methods to read integers, please expose a method that makes the byte order explicit such
|
||||
// as readIntLE (little endian).
|
||||
|
||||
// Also, please ***NEVER*** expose any method that writes using Unsafe, this is too dangerous
|
||||
|
||||
private static long readLong(byte[] src, int offset) {
|
||||
return UNSAFE.getLong(src, BYTE_ARRAY_OFFSET + offset);
|
||||
}
|
||||
|
||||
private static int readInt(byte[] src, int offset) {
|
||||
return UNSAFE.getInt(src, BYTE_ARRAY_OFFSET + offset);
|
||||
}
|
||||
|
||||
private static short readShort(byte[] src, int offset) {
|
||||
return UNSAFE.getShort(src, BYTE_ARRAY_OFFSET + offset);
|
||||
}
|
||||
|
||||
private static byte readByte(byte[] src, int offset) {
|
||||
return UNSAFE.getByte(src, BYTE_ARRAY_OFFSET + BYTE_ARRAY_SCALE * offset);
|
||||
}
|
||||
|
||||
/** Compare the two given {@link BytesRef}s for equality. */
|
||||
public static boolean equals(BytesRef b1, BytesRef b2) {
|
||||
if (b1.length != b2.length) {
|
||||
return false;
|
||||
}
|
||||
return equals(b1.bytes, b1.offset, b2.bytes, b2.offset, b1.length);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare <code>b1[offset1:offset1+length)</code>against <code>b1[offset2:offset2+length)</code>.
|
||||
*/
|
||||
public static boolean equals(byte[] b1, int offset1, byte[] b2, int offset2, int length) {
|
||||
int o1 = offset1;
|
||||
int o2 = offset2;
|
||||
int len = length;
|
||||
while (len >= 8) {
|
||||
if (readLong(b1, o1) != readLong(b2, o2)) {
|
||||
return false;
|
||||
}
|
||||
len -= 8;
|
||||
o1 += 8;
|
||||
o2 += 8;
|
||||
}
|
||||
if (len >= 4) {
|
||||
if (readInt(b1, o1) != readInt(b2, o2)) {
|
||||
return false;
|
||||
}
|
||||
len -= 4;
|
||||
o1 += 4;
|
||||
o2 += 4;
|
||||
}
|
||||
if (len >= 2) {
|
||||
if (readShort(b1, o1) != readShort(b2, o2)) {
|
||||
return false;
|
||||
}
|
||||
len -= 2;
|
||||
o1 += 2;
|
||||
o2 += 2;
|
||||
}
|
||||
if (len == 1) {
|
||||
if (readByte(b1, o1) != readByte(b2, o2)) {
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
assert len == 0;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a long using little endian byte order.
|
||||
*/
|
||||
public static long readLongLE(byte[] src, int offset) {
|
||||
long value = readLong(src, offset);
|
||||
if (ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN) {
|
||||
value = Long.reverseBytes(value);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read an int using little endian byte order.
|
||||
*/
|
||||
public static int readIntLE(byte[] src, int offset) {
|
||||
int value = readInt(src, offset);
|
||||
if (ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN) {
|
||||
value = Integer.reverseBytes(value);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -31,8 +31,8 @@ import org.elasticsearch.common.lease.Releasable;
|
|||
import org.elasticsearch.common.lease.Releasables;
|
||||
import org.elasticsearch.common.util.BigArrays;
|
||||
import org.elasticsearch.common.util.ByteArray;
|
||||
import org.elasticsearch.common.util.ByteUtils;
|
||||
import org.elasticsearch.common.util.IntArray;
|
||||
import org.elasticsearch.common.util.UnsafeUtils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
|
|
@ -438,7 +438,7 @@ public final class HyperLogLogPlusPlus implements Releasable {
|
|||
|
||||
private int get(long bucket, int index) {
|
||||
runLens.get(index(bucket, index), 4, readSpare);
|
||||
return UnsafeUtils.readIntLE(readSpare.bytes, readSpare.offset);
|
||||
return ByteUtils.readIntLE(readSpare.bytes, readSpare.offset);
|
||||
}
|
||||
|
||||
private void set(long bucket, int index, int value) {
|
||||
|
|
|
|||
|
|
@ -1,140 +0,0 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.benchmark.common.util;
|
||||
|
||||
import com.carrotsearch.randomizedtesting.generators.RandomInts;
|
||||
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.elasticsearch.common.util.UnsafeUtils;
|
||||
|
||||
import java.util.Random;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
public class BytesRefComparisonsBenchmark {
|
||||
|
||||
private static final Random R = new Random(0);
|
||||
private static final int ITERS = 100;
|
||||
|
||||
// To avoid JVM optimizations
|
||||
@SuppressWarnings("unused")
|
||||
private static boolean DUMMY;
|
||||
|
||||
enum Comparator {
|
||||
SAFE {
|
||||
boolean compare(BytesRef b1, BytesRef b2) {
|
||||
return b1.bytesEquals(b2);
|
||||
}
|
||||
},
|
||||
UNSAFE {
|
||||
@Override
|
||||
boolean compare(BytesRef b1, BytesRef b2) {
|
||||
return UnsafeUtils.equals(b1, b2);
|
||||
}
|
||||
};
|
||||
abstract boolean compare(BytesRef b1, BytesRef b2);
|
||||
}
|
||||
|
||||
private static BytesRef[] buildBytesRefs(int minLen, int maxLen, int count, int uniqueCount) {
|
||||
final BytesRef[] uniqueRefs = new BytesRef[uniqueCount];
|
||||
for (int i = 0; i < uniqueCount; ++i) {
|
||||
final int len = RandomInts.randomIntBetween(R, minLen, maxLen);
|
||||
final byte[] bytes = new byte[len];
|
||||
for (int j = 0; j < bytes.length; ++j) {
|
||||
bytes[j] = (byte) R.nextInt(2); // so that some instances have common prefixes
|
||||
}
|
||||
uniqueRefs[i] = new BytesRef(bytes);
|
||||
}
|
||||
final BytesRef[] result = new BytesRef[count];
|
||||
for (int i = 0; i < count; ++i) {
|
||||
result[i] = RandomPicks.randomFrom(R, uniqueRefs);
|
||||
}
|
||||
int totalLen = 0;
|
||||
for (BytesRef b : result) {
|
||||
totalLen += b.length;
|
||||
}
|
||||
final byte[] data = new byte[totalLen];
|
||||
int offset = 0;
|
||||
for (int i = 0; i < count; ++i) {
|
||||
final BytesRef b = result[i];
|
||||
System.arraycopy(b.bytes, b.offset, data, offset, b.length);
|
||||
result[i] = new BytesRef(data, offset, b.length);
|
||||
offset += b.length;
|
||||
}
|
||||
if (offset != totalLen) {
|
||||
throw new AssertionError();
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static long bench(Comparator comparator, BytesRef[] refs, int iters) {
|
||||
boolean xor = false;
|
||||
final long start = System.nanoTime();
|
||||
for (int iter = 0; iter < iters; ++iter) {
|
||||
for (int i = 0; i < refs.length; ++i) {
|
||||
for (int j = i + 1; j < refs.length; ++j) {
|
||||
xor ^= comparator.compare(refs[i], refs[j]);
|
||||
}
|
||||
}
|
||||
}
|
||||
DUMMY = xor;
|
||||
return System.nanoTime() - start;
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws InterruptedException {
|
||||
// warmup
|
||||
BytesRef[] bytes = buildBytesRefs(2, 20, 1000, 100);
|
||||
final long start = System.nanoTime();
|
||||
while (System.nanoTime() - start < TimeUnit.SECONDS.toNanos(10)) {
|
||||
for (Comparator comparator : Comparator.values()) {
|
||||
bench(comparator, bytes, 1);
|
||||
}
|
||||
}
|
||||
|
||||
System.out.println("## Various lengths");
|
||||
// make sure GC doesn't hurt results
|
||||
System.gc();
|
||||
Thread.sleep(2000);
|
||||
for (Comparator comparator : Comparator.values()) {
|
||||
bench(comparator, bytes, ITERS);
|
||||
}
|
||||
for (int i = 0; i < 3; ++i) {
|
||||
for (Comparator comparator : Comparator.values()) {
|
||||
System.out.println(comparator + " " + new TimeValue(bench(comparator, bytes, ITERS), TimeUnit.NANOSECONDS));
|
||||
}
|
||||
}
|
||||
|
||||
for (int len = 2; len <= 20; ++len) {
|
||||
System.out.println("## Length = " + len);
|
||||
bytes = buildBytesRefs(len, len, 1000, 100);
|
||||
System.gc();
|
||||
Thread.sleep(2000);
|
||||
for (Comparator comparator : Comparator.values()) {
|
||||
bench(comparator, bytes, ITERS);
|
||||
}
|
||||
for (int i = 0; i < 3; ++i) {
|
||||
for (Comparator comparator : Comparator.values()) {
|
||||
System.out.println(comparator + " " + new TimeValue(bench(comparator, bytes, ITERS), TimeUnit.NANOSECONDS));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -37,13 +37,13 @@ public class BytesReferenceTests extends ElasticsearchTestCase {
|
|||
final BytesArray b1 = new BytesArray(array1, offset1, len);
|
||||
final BytesArray b2 = new BytesArray(array2, offset2, len);
|
||||
assertTrue(BytesReference.Helper.bytesEqual(b1, b2));
|
||||
assertTrue(BytesReference.Helper.slowBytesEquals(b1, b2));
|
||||
assertTrue(BytesReference.Helper.bytesEquals(b1, b2));
|
||||
assertEquals(Arrays.hashCode(b1.toBytes()), b1.hashCode());
|
||||
assertEquals(BytesReference.Helper.bytesHashCode(b1), BytesReference.Helper.slowHashCode(b2));
|
||||
|
||||
// test same instance
|
||||
assertTrue(BytesReference.Helper.bytesEqual(b1, b1));
|
||||
assertTrue(BytesReference.Helper.slowBytesEquals(b1, b1));
|
||||
assertTrue(BytesReference.Helper.bytesEquals(b1, b1));
|
||||
assertEquals(BytesReference.Helper.bytesHashCode(b1), BytesReference.Helper.slowHashCode(b1));
|
||||
|
||||
if (len > 0) {
|
||||
|
|
@ -54,7 +54,7 @@ public class BytesReferenceTests extends ElasticsearchTestCase {
|
|||
// test changed bytes
|
||||
array1[offset1 + randomInt(len - 1)] += 13;
|
||||
assertFalse(BytesReference.Helper.bytesEqual(b1, b2));
|
||||
assertFalse(BytesReference.Helper.slowBytesEquals(b1, b2));
|
||||
assertFalse(BytesReference.Helper.bytesEquals(b1, b2));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue