HADOOP-14424. Add CRC32C performance test. Contributed by LiXin Ge.
(cherry picked from commit 75043d3429)
This commit is contained in:
Masatake Iwasaki
parent
3106f256e2
commit
023dcdc73d
|
|
@ -33,7 +33,7 @@ import org.apache.hadoop.fs.ChecksumException;
|
|||
import org.apache.log4j.Level;
|
||||
|
||||
/**
|
||||
* Performance tests to compare performance of Crc32 implementations
|
||||
* Performance tests to compare performance of Crc32|Crc32C implementations
|
||||
* This can be run from the command line with:
|
||||
*
|
||||
* java -cp path/to/test/classes:path/to/common/classes \
|
||||
|
|
@ -43,17 +43,21 @@ import org.apache.log4j.Level;
|
|||
*
|
||||
* hadoop org.apache.hadoop.util.Crc32PerformanceTest
|
||||
*
|
||||
* If any argument is provided, this test will run with non-directly buffer.
|
||||
*
|
||||
* The output is in JIRA table format.
|
||||
*/
|
||||
public class Crc32PerformanceTest {
|
||||
static final int MB = 1024 * 1024;
|
||||
|
||||
static interface Crc32 {
|
||||
interface Crc32 {
|
||||
|
||||
public void verifyChunked(ByteBuffer data, int bytesPerCrc, ByteBuffer crcs,
|
||||
void verifyChunked(ByteBuffer data, int bytesPerCrc, ByteBuffer crcs,
|
||||
String filename, long basePos) throws ChecksumException;
|
||||
|
||||
static final class Native implements Crc32 {
|
||||
DataChecksum.Type crcType();
|
||||
|
||||
final class Native implements Crc32 {
|
||||
@Override
|
||||
public void verifyChunked(ByteBuffer data, int bytesPerSum,
|
||||
ByteBuffer sums, String fileName, long basePos)
|
||||
|
|
@ -61,40 +65,91 @@ public class Crc32PerformanceTest {
|
|||
NativeCrc32.verifyChunkedSums(bytesPerSum, DataChecksum.Type.CRC32.id,
|
||||
sums, data, fileName, basePos);
|
||||
}
|
||||
|
||||
@Override
|
||||
public DataChecksum.Type crcType() {
|
||||
return DataChecksum.Type.CRC32;
|
||||
}
|
||||
}
|
||||
|
||||
final class NativeC implements Crc32 {
|
||||
@Override
|
||||
public void verifyChunked(ByteBuffer data, int bytesPerSum,
|
||||
ByteBuffer sums, String fileName, long basePos)
|
||||
throws ChecksumException {
|
||||
|
||||
static abstract class AbstractCrc32<T extends Checksum> implements Crc32 {
|
||||
if (data.isDirect()) {
|
||||
NativeCrc32.verifyChunkedSums(bytesPerSum,
|
||||
DataChecksum.Type.CRC32C.id, sums, data, fileName, basePos);
|
||||
} else {
|
||||
final int dataOffset = data.arrayOffset() + data.position();
|
||||
final int crcsOffset = sums.arrayOffset() + sums.position();
|
||||
NativeCrc32.verifyChunkedSumsByteArray(bytesPerSum,
|
||||
DataChecksum.Type.CRC32C.id, sums.array(), crcsOffset,
|
||||
data.array(), dataOffset, data.remaining(), fileName, basePos);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public DataChecksum.Type crcType() {
|
||||
return DataChecksum.Type.CRC32C;
|
||||
}
|
||||
}
|
||||
|
||||
abstract class AbstractCrc32<T extends Checksum> implements Crc32 {
|
||||
abstract T newAlgorithm();
|
||||
|
||||
@Override
|
||||
public void verifyChunked(ByteBuffer data, int bytesPerCrc,
|
||||
ByteBuffer crcs, String filename, long basePos)
|
||||
ByteBuffer sums, String filename, long basePos)
|
||||
throws ChecksumException {
|
||||
final Checksum algorithm = newAlgorithm();
|
||||
if (data.hasArray() && crcs.hasArray()) {
|
||||
DataChecksum.verifyChunked(DataChecksum.Type.CRC32, algorithm,
|
||||
data.array(), data.position(), data.remaining(), bytesPerCrc,
|
||||
crcs.array(), crcs.position(), filename, basePos);
|
||||
final DataChecksum.Type type = crcType();
|
||||
if (data.hasArray() && sums.hasArray()) {
|
||||
DataChecksum.verifyChunked(type, algorithm, data.array(),
|
||||
data.position(), data.remaining(), bytesPerCrc, sums.array(),
|
||||
sums.position(), filename, basePos);
|
||||
} else {
|
||||
DataChecksum.verifyChunked(DataChecksum.Type.CRC32, algorithm,
|
||||
data, bytesPerCrc, crcs, filename, basePos);
|
||||
DataChecksum.verifyChunked(type, algorithm, data, bytesPerCrc,
|
||||
sums, filename, basePos);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static final class Zip extends AbstractCrc32<CRC32> {
|
||||
final class Zip extends AbstractCrc32<CRC32> {
|
||||
@Override
|
||||
public CRC32 newAlgorithm() {
|
||||
return new CRC32();
|
||||
}
|
||||
|
||||
@Override
|
||||
public DataChecksum.Type crcType() {
|
||||
return DataChecksum.Type.CRC32;
|
||||
}
|
||||
}
|
||||
|
||||
static final class PureJava extends AbstractCrc32<PureJavaCrc32> {
|
||||
final class PureJava extends AbstractCrc32<PureJavaCrc32> {
|
||||
@Override
|
||||
public PureJavaCrc32 newAlgorithm() {
|
||||
return new PureJavaCrc32();
|
||||
}
|
||||
|
||||
@Override
|
||||
public DataChecksum.Type crcType() {
|
||||
return DataChecksum.Type.CRC32;
|
||||
}
|
||||
}
|
||||
|
||||
final class PureJavaC extends AbstractCrc32<PureJavaCrc32C> {
|
||||
@Override
|
||||
public PureJavaCrc32C newAlgorithm() {
|
||||
return new PureJavaCrc32C();
|
||||
}
|
||||
|
||||
@Override
|
||||
public DataChecksum.Type crcType() {
|
||||
return DataChecksum.Type.CRC32C;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -114,9 +169,13 @@ public class Crc32PerformanceTest {
|
|||
|
||||
crcs.add(Crc32.Zip.class);
|
||||
crcs.add(Crc32.PureJava.class);
|
||||
crcs.add(Crc32.PureJavaC.class);
|
||||
|
||||
if (direct && NativeCrc32.isAvailable()) {
|
||||
crcs.add(Crc32.Native.class);
|
||||
if (NativeCrc32.isAvailable()) {
|
||||
if (direct) {
|
||||
crcs.add(Crc32.Native.class);
|
||||
}
|
||||
crcs.add(Crc32.NativeC.class);
|
||||
((Log4JLogger)LogFactory.getLog(NativeCodeLoader.class))
|
||||
.getLogger().setLevel(Level.ALL);
|
||||
}
|
||||
|
|
@ -131,13 +190,18 @@ public class Crc32PerformanceTest {
|
|||
out.printf("Elapsed %.1fs\n", secondsElapsed(startTime));
|
||||
}
|
||||
|
||||
public static void main(String args[]) throws Exception {
|
||||
new Crc32PerformanceTest(64, 5, true).run();
|
||||
public static void main(String[] args) throws Exception {
|
||||
boolean isdirect = true;
|
||||
|
||||
if (args.length > 0) {
|
||||
isdirect = false;
|
||||
}
|
||||
new Crc32PerformanceTest(64, 5, isdirect).run();
|
||||
}
|
||||
|
||||
private static void printCell(String s, int width, PrintStream out) {
|
||||
private static void printCell(String s, int width, PrintStream outCrc) {
|
||||
final int w = s.length() > width? s.length(): width;
|
||||
out.printf(" %" + w + "s |", s);
|
||||
outCrc.printf(" %" + w + "s |", s);
|
||||
}
|
||||
|
||||
private ByteBuffer allocateByteBuffer(int length) {
|
||||
|
|
@ -155,16 +219,29 @@ public class Crc32PerformanceTest {
|
|||
return dataBufs;
|
||||
}
|
||||
|
||||
private ByteBuffer computeCrc(ByteBuffer dataBufs, int bytePerCrc) {
|
||||
private ByteBuffer computeCrc(ByteBuffer dataBufs, int bytePerCrc,
|
||||
DataChecksum.Type type) {
|
||||
final int size = 4 * (dataBufs.remaining() - 1) / bytePerCrc + 1;
|
||||
final ByteBuffer crcBufs = allocateByteBuffer(size);
|
||||
final DataChecksum checksum = DataChecksum.newDataChecksum(
|
||||
DataChecksum.Type.CRC32, bytePerCrc);
|
||||
type, bytePerCrc);
|
||||
checksum.calculateChunkedSums(dataBufs, crcBufs);
|
||||
return crcBufs;
|
||||
}
|
||||
|
||||
private void doBench(final List<Class<? extends Crc32>> crcs)
|
||||
private ByteBuffer computeCrc(Class<? extends Crc32> clazz,
|
||||
ByteBuffer dataBufs, int bytePerCrc) throws Exception {
|
||||
final Constructor<? extends Crc32> ctor = clazz.getConstructor();
|
||||
final Crc32 crc = ctor.newInstance();
|
||||
final int size = 4 * (dataBufs.remaining() - 1) / bytePerCrc + 1;
|
||||
final ByteBuffer crcBufs = allocateByteBuffer(size);
|
||||
final DataChecksum checksum = DataChecksum.newDataChecksum(
|
||||
crc.crcType(), bytePerCrc);
|
||||
checksum.calculateChunkedSums(dataBufs, crcBufs);
|
||||
return crcBufs;
|
||||
}
|
||||
|
||||
private void doBench(final List<Class<? extends Crc32>> crcTargets)
|
||||
throws Exception {
|
||||
final ByteBuffer[] dataBufs = new ByteBuffer[16];
|
||||
for(int i = 0; i < dataBufs.length; i++) {
|
||||
|
|
@ -176,9 +253,9 @@ public class Crc32PerformanceTest {
|
|||
out.printf(" (bpc: byte-per-crc in MB/sec; #T: #Theads)\n");
|
||||
|
||||
// Warm up implementations to get jit going.
|
||||
final ByteBuffer[] crc32 = {computeCrc(dataBufs[0], 32)};
|
||||
final ByteBuffer[] crc512 = {computeCrc(dataBufs[0], 512)};
|
||||
for (Class<? extends Crc32> c : crcs) {
|
||||
for (Class<? extends Crc32> c : crcTargets) {
|
||||
final ByteBuffer[] crc32 = {computeCrc(c, dataBufs[0], 32)};
|
||||
final ByteBuffer[] crc512 = {computeCrc(c, dataBufs[0], 512)};
|
||||
doBench(c, 1, dataBufs, crc32, 32);
|
||||
doBench(c, 1, dataBufs, crc512, 512);
|
||||
}
|
||||
|
|
@ -189,57 +266,69 @@ public class Crc32PerformanceTest {
|
|||
}
|
||||
}
|
||||
|
||||
private void doBench(final List<Class<? extends Crc32>> crcs,
|
||||
final ByteBuffer[] dataBufs, final int bytePerCrc, final PrintStream out)
|
||||
private void doBench(final List<Class<? extends Crc32>> crcTargets,
|
||||
final ByteBuffer[] dataBufs, final int bytePerCrc,
|
||||
final PrintStream outCrc)
|
||||
throws Exception {
|
||||
final ByteBuffer[] crcBufs = new ByteBuffer[dataBufs.length];
|
||||
for(int i = 0; i < crcBufs.length; i++) {
|
||||
crcBufs[i] = computeCrc(dataBufs[i], bytePerCrc);
|
||||
final ByteBuffer[] crcBufsC = new ByteBuffer[dataBufs.length];
|
||||
for(int i = 0; i < dataBufs.length; i++) {
|
||||
crcBufs[i] = computeCrc(dataBufs[i], bytePerCrc,
|
||||
DataChecksum.Type.CRC32);
|
||||
crcBufsC[i] = computeCrc(dataBufs[i], bytePerCrc,
|
||||
DataChecksum.Type.CRC32C);
|
||||
}
|
||||
|
||||
final String numBytesStr = " bpc ";
|
||||
final String numThreadsStr = "#T";
|
||||
final String diffStr = "% diff";
|
||||
|
||||
out.print('|');
|
||||
printCell(numBytesStr, 0, out);
|
||||
printCell(numThreadsStr, 0, out);
|
||||
for (int i = 0; i < crcs.size(); i++) {
|
||||
final Class<? extends Crc32> c = crcs.get(i);
|
||||
out.print('|');
|
||||
printCell(c.getSimpleName(), 8, out);
|
||||
for(int j = 0; j < i; j++) {
|
||||
printCell(diffStr, diffStr.length(), out);
|
||||
outCrc.print('|');
|
||||
printCell(numBytesStr, 0, outCrc);
|
||||
printCell(numThreadsStr, 0, outCrc);
|
||||
for (int i = 0; i < crcTargets.size(); i++) {
|
||||
final Class<? extends Crc32> c = crcTargets.get(i);
|
||||
outCrc.print('|');
|
||||
printCell(c.getSimpleName(), 8, outCrc);
|
||||
if (i > 0) {
|
||||
printCell(diffStr, diffStr.length(), outCrc);
|
||||
}
|
||||
}
|
||||
out.printf("\n");
|
||||
outCrc.printf("\n");
|
||||
|
||||
for(int numThreads = 1; numThreads <= dataBufs.length; numThreads <<= 1) {
|
||||
out.printf("|");
|
||||
printCell(String.valueOf(bytePerCrc), numBytesStr.length(), out);
|
||||
printCell(String.valueOf(numThreads), numThreadsStr.length(), out);
|
||||
outCrc.printf("|");
|
||||
printCell(String.valueOf(bytePerCrc), numBytesStr.length(), outCrc);
|
||||
printCell(String.valueOf(numThreads), numThreadsStr.length(), outCrc);
|
||||
|
||||
final List<BenchResult> previous = new ArrayList<BenchResult>();
|
||||
for(Class<? extends Crc32> c : crcs) {
|
||||
for(Class<? extends Crc32> c : crcTargets) {
|
||||
System.gc();
|
||||
|
||||
final BenchResult result = doBench(c, numThreads, dataBufs, crcBufs,
|
||||
bytePerCrc);
|
||||
final BenchResult result;
|
||||
final Constructor<? extends Crc32> ctor = c.getConstructor();
|
||||
final Crc32 crc = ctor.newInstance();
|
||||
if (crc.crcType() == DataChecksum.Type.CRC32) {
|
||||
result = doBench(c, numThreads, dataBufs, crcBufs, bytePerCrc);
|
||||
} else {
|
||||
result = doBench(c, numThreads, dataBufs, crcBufsC, bytePerCrc);
|
||||
}
|
||||
printCell(String.format("%9.1f", result.mbps),
|
||||
c.getSimpleName().length() + 1, out);
|
||||
c.getSimpleName().length() + 1, outCrc);
|
||||
|
||||
//compare result with previous
|
||||
for(BenchResult p : previous) {
|
||||
//compare result with the last previous.
|
||||
final int size = previous.size();
|
||||
if (size > 0) {
|
||||
BenchResult p = previous.get(size - 1);
|
||||
final double diff = (result.mbps - p.mbps) / p.mbps * 100;
|
||||
printCell(String.format("%5.1f%%", diff), diffStr.length(), out);
|
||||
printCell(String.format("%5.1f%%", diff), diffStr.length(), outCrc);
|
||||
}
|
||||
previous.add(result);
|
||||
}
|
||||
out.printf("\n");
|
||||
outCrc.printf("\n");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private BenchResult doBench(Class<? extends Crc32> clazz,
|
||||
final int numThreads, final ByteBuffer[] dataBufs,
|
||||
final ByteBuffer[] crcBufs, final int bytePerCrc)
|
||||
|
|
@ -248,36 +337,34 @@ public class Crc32PerformanceTest {
|
|||
final Thread[] threads = new Thread[numThreads];
|
||||
final BenchResult[] results = new BenchResult[threads.length];
|
||||
|
||||
{
|
||||
final Constructor<? extends Crc32> ctor = clazz.getConstructor();
|
||||
final Constructor<? extends Crc32> ctor = clazz.getConstructor();
|
||||
|
||||
for(int i = 0; i < threads.length; i++) {
|
||||
final Crc32 crc = ctor.newInstance();
|
||||
final long byteProcessed = dataBufs[i].remaining() * trials;
|
||||
final int index = i;
|
||||
threads[i] = new Thread() {
|
||||
@Override
|
||||
public void run() {
|
||||
final long startTime = System.nanoTime();
|
||||
for (int i = 0; i < trials; i++) {
|
||||
dataBufs[index].mark();
|
||||
crcBufs[index].mark();
|
||||
try {
|
||||
crc.verifyChunked(dataBufs[index], bytePerCrc, crcBufs[index],
|
||||
crc.getClass().getSimpleName(), dataBufs[index].position());
|
||||
} catch (Throwable t) {
|
||||
results[index] = new BenchResult(t);
|
||||
return;
|
||||
} finally {
|
||||
dataBufs[index].reset();
|
||||
crcBufs[index].reset();
|
||||
}
|
||||
for(int i = 0; i < threads.length; i++) {
|
||||
final Crc32 crc = ctor.newInstance();
|
||||
final long byteProcessed = dataBufs[i].remaining() * trials;
|
||||
final int index = i;
|
||||
threads[i] = new Thread() {
|
||||
@Override
|
||||
public void run() {
|
||||
final long startTime = System.nanoTime();
|
||||
for (int i = 0; i < trials; i++) {
|
||||
dataBufs[index].mark();
|
||||
crcBufs[index].mark();
|
||||
try {
|
||||
crc.verifyChunked(dataBufs[index], bytePerCrc, crcBufs[index],
|
||||
crc.getClass().getSimpleName(), dataBufs[index].position());
|
||||
} catch (Throwable t) {
|
||||
results[index] = new BenchResult(t);
|
||||
return;
|
||||
} finally {
|
||||
dataBufs[index].reset();
|
||||
crcBufs[index].reset();
|
||||
}
|
||||
final double secsElapsed = secondsElapsed(startTime);
|
||||
results[index] = new BenchResult(byteProcessed/secsElapsed/MB);
|
||||
}
|
||||
};
|
||||
}
|
||||
final double secsElapsed = secondsElapsed(startTime);
|
||||
results[index] = new BenchResult(byteProcessed/secsElapsed/MB);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
for(Thread t : threads) {
|
||||
|
|
@ -295,7 +382,7 @@ public class Crc32PerformanceTest {
|
|||
}
|
||||
|
||||
private static class BenchResult {
|
||||
/** Speed (MB per second) */
|
||||
/** Speed (MB per second). */
|
||||
final double mbps;
|
||||
final Throwable thrown;
|
||||
|
||||
|
|
@ -321,7 +408,7 @@ public class Crc32PerformanceTest {
|
|||
return (System.nanoTime() - startTime) / 1000000000.0d;
|
||||
}
|
||||
|
||||
static void printSystemProperties(PrintStream out) {
|
||||
static void printSystemProperties(PrintStream outCrc) {
|
||||
final String[] names = {
|
||||
"java.version",
|
||||
"java.runtime.name",
|
||||
|
|
@ -344,7 +431,7 @@ public class Crc32PerformanceTest {
|
|||
|
||||
final Properties p = System.getProperties();
|
||||
for(String n : names) {
|
||||
out.printf("%" + max + "s = %s\n", n, p.getProperty(n));
|
||||
outCrc.printf("%" + max + "s = %s\n", n, p.getProperty(n));
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue