HADOOP-14424. Add CRC32C performance test. Contributed by LiXin Ge.

(cherry picked from commit 75043d3429)

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/Crc32PerformanceTest.java

@@ -33,7 +33,7 @@ import org.apache.hadoop.fs.ChecksumException;
 import org.apache.log4j.Level;
 
 /**
- * Performance tests to compare performance of Crc32 implementations
+ * Performance tests to compare performance of Crc32|Crc32C implementations
  * This can be run from the command line with:
  *
  *   java -cp path/to/test/classes:path/to/common/classes \
@@ -43,17 +43,21 @@ import org.apache.log4j.Level;
  *
  *  hadoop org.apache.hadoop.util.Crc32PerformanceTest
  *
+ * If any argument is provided, this test will run with non-directly buffer.
+ *
  * The output is in JIRA table format.
  */
 public class Crc32PerformanceTest {
   static final int MB = 1024 * 1024;
 
-  static interface Crc32 {
+  interface Crc32 {
 
-    public void verifyChunked(ByteBuffer data, int bytesPerCrc, ByteBuffer crcs,
+    void verifyChunked(ByteBuffer data, int bytesPerCrc, ByteBuffer crcs,
         String filename, long basePos) throws ChecksumException;
 
-    static final class Native implements Crc32 {
+    DataChecksum.Type crcType();
+
+    final class Native implements Crc32 {
       @Override
       public void verifyChunked(ByteBuffer data, int bytesPerSum,
           ByteBuffer sums, String fileName, long basePos)
@@ -61,40 +65,91 @@ public class Crc32PerformanceTest {
         NativeCrc32.verifyChunkedSums(bytesPerSum, DataChecksum.Type.CRC32.id,
             sums, data, fileName, basePos);
       }
+
+      @Override
+      public DataChecksum.Type crcType() {
+        return DataChecksum.Type.CRC32;
+      }
     }
 
+    final class NativeC implements Crc32 {
+      @Override
+      public void verifyChunked(ByteBuffer data, int bytesPerSum,
+          ByteBuffer sums, String fileName, long basePos)
+              throws ChecksumException {
 
-    static abstract class AbstractCrc32<T extends Checksum> implements Crc32 {
+        if (data.isDirect()) {
+          NativeCrc32.verifyChunkedSums(bytesPerSum,
+                  DataChecksum.Type.CRC32C.id, sums, data, fileName, basePos);
+        } else {
+          final int dataOffset = data.arrayOffset() + data.position();
+          final int crcsOffset = sums.arrayOffset() + sums.position();
+          NativeCrc32.verifyChunkedSumsByteArray(bytesPerSum,
+              DataChecksum.Type.CRC32C.id, sums.array(), crcsOffset,
+              data.array(), dataOffset, data.remaining(), fileName, basePos);
+        }
+      }
+
+      @Override
+      public DataChecksum.Type crcType() {
+        return DataChecksum.Type.CRC32C;
+      }
+    }
+
+    abstract class AbstractCrc32<T extends Checksum> implements Crc32 {
       abstract T newAlgorithm();
 
       @Override
       public void verifyChunked(ByteBuffer data, int bytesPerCrc,
-          ByteBuffer crcs, String filename, long basePos)
+          ByteBuffer sums, String filename, long basePos)
               throws ChecksumException {
         final Checksum algorithm = newAlgorithm();
-        if (data.hasArray() && crcs.hasArray()) {
+        final DataChecksum.Type type = crcType();
-          DataChecksum.verifyChunked(DataChecksum.Type.CRC32, algorithm,
+        if (data.hasArray() && sums.hasArray()) {
-              data.array(), data.position(), data.remaining(), bytesPerCrc,
+          DataChecksum.verifyChunked(type, algorithm, data.array(),
-              crcs.array(), crcs.position(), filename, basePos);
+              data.position(), data.remaining(), bytesPerCrc, sums.array(),
+              sums.position(), filename, basePos);
         } else {
-          DataChecksum.verifyChunked(DataChecksum.Type.CRC32, algorithm,
+          DataChecksum.verifyChunked(type, algorithm, data, bytesPerCrc,
-              data, bytesPerCrc, crcs, filename, basePos);
+              sums, filename, basePos);
         }
       }
     }
 
-    static final class Zip extends AbstractCrc32<CRC32> {
+    final class Zip extends AbstractCrc32<CRC32> {
       @Override
       public CRC32 newAlgorithm() {
         return new CRC32();
       }
+
+      @Override
+      public DataChecksum.Type crcType() {
+        return DataChecksum.Type.CRC32;
+      }
     }
 
-    static final class PureJava extends AbstractCrc32<PureJavaCrc32> {
+    final class PureJava extends AbstractCrc32<PureJavaCrc32> {
       @Override
       public PureJavaCrc32 newAlgorithm() {
         return new PureJavaCrc32();
       }
+
+      @Override
+      public DataChecksum.Type crcType() {
+        return DataChecksum.Type.CRC32;
+      }
+    }
+
+    final class PureJavaC extends AbstractCrc32<PureJavaCrc32C> {
+      @Override
+      public PureJavaCrc32C newAlgorithm() {
+        return new PureJavaCrc32C();
+      }
+
+      @Override
+      public DataChecksum.Type crcType() {
+        return DataChecksum.Type.CRC32C;
+      }
     }
   }
 
@@ -114,9 +169,13 @@ public class Crc32PerformanceTest {
 
     crcs.add(Crc32.Zip.class);
     crcs.add(Crc32.PureJava.class);
+    crcs.add(Crc32.PureJavaC.class);
 
-    if (direct && NativeCrc32.isAvailable()) {
+    if (NativeCrc32.isAvailable()) {
+      if (direct) {
         crcs.add(Crc32.Native.class);
+      }
+      crcs.add(Crc32.NativeC.class);
       ((Log4JLogger)LogFactory.getLog(NativeCodeLoader.class))
           .getLogger().setLevel(Level.ALL);
     }
@@ -131,13 +190,18 @@ public class Crc32PerformanceTest {
     out.printf("Elapsed %.1fs\n", secondsElapsed(startTime));
   }
 
-  public static void main(String args[]) throws Exception {
+  public static void main(String[] args) throws Exception {
-    new Crc32PerformanceTest(64, 5, true).run();
+    boolean isdirect = true;
+
+    if (args.length > 0) {
+      isdirect = false;
+    }
+    new Crc32PerformanceTest(64, 5, isdirect).run();
   }
 
-  private static void printCell(String s, int width, PrintStream out) {
+  private static void printCell(String s, int width, PrintStream outCrc) {
     final int w = s.length() > width? s.length(): width;
-    out.printf(" %" + w + "s |", s);
+    outCrc.printf(" %" + w + "s |", s);
   }
 
   private ByteBuffer allocateByteBuffer(int length) {
@@ -155,16 +219,29 @@ public class Crc32PerformanceTest {
     return dataBufs;
   }
 
-  private ByteBuffer computeCrc(ByteBuffer dataBufs, int bytePerCrc) {
+  private ByteBuffer computeCrc(ByteBuffer dataBufs, int bytePerCrc,
+      DataChecksum.Type type) {
     final int size = 4 * (dataBufs.remaining() - 1) / bytePerCrc + 1;
     final ByteBuffer crcBufs = allocateByteBuffer(size);
     final DataChecksum checksum = DataChecksum.newDataChecksum(
-        DataChecksum.Type.CRC32, bytePerCrc);
+        type, bytePerCrc);
     checksum.calculateChunkedSums(dataBufs, crcBufs);
     return crcBufs;
   }
 
-  private void doBench(final List<Class<? extends Crc32>> crcs)
+  private ByteBuffer computeCrc(Class<? extends Crc32> clazz,
+      ByteBuffer dataBufs, int bytePerCrc) throws Exception {
+    final Constructor<? extends Crc32> ctor = clazz.getConstructor();
+    final Crc32 crc = ctor.newInstance();
+    final int size = 4 * (dataBufs.remaining() - 1) / bytePerCrc + 1;
+    final ByteBuffer crcBufs = allocateByteBuffer(size);
+    final DataChecksum checksum = DataChecksum.newDataChecksum(
+        crc.crcType(), bytePerCrc);
+    checksum.calculateChunkedSums(dataBufs, crcBufs);
+    return crcBufs;
+  }
+
+  private void doBench(final List<Class<? extends Crc32>> crcTargets)
       throws Exception {
     final ByteBuffer[] dataBufs = new ByteBuffer[16];
     for(int i = 0; i < dataBufs.length; i++) {
@@ -176,9 +253,9 @@ public class Crc32PerformanceTest {
     out.printf(" (bpc: byte-per-crc in MB/sec; #T: #Theads)\n");
 
     // Warm up implementations to get jit going.
-    final ByteBuffer[] crc32 = {computeCrc(dataBufs[0], 32)};
+    for (Class<? extends Crc32> c : crcTargets) {
-    final ByteBuffer[] crc512 = {computeCrc(dataBufs[0], 512)};
+      final ByteBuffer[] crc32 = {computeCrc(c, dataBufs[0], 32)};
-    for (Class<? extends Crc32> c : crcs) {
+      final ByteBuffer[] crc512 = {computeCrc(c, dataBufs[0], 512)};
       doBench(c, 1, dataBufs, crc32, 32);
       doBench(c, 1, dataBufs, crc512, 512);
     }
@@ -189,57 +266,69 @@ public class Crc32PerformanceTest {
     }
   }
 
-  private void doBench(final List<Class<? extends Crc32>> crcs,
+  private void doBench(final List<Class<? extends Crc32>> crcTargets,
-      final ByteBuffer[] dataBufs, final int bytePerCrc, final PrintStream out)
+      final ByteBuffer[] dataBufs, final int bytePerCrc,
+      final PrintStream outCrc)
           throws Exception {
     final ByteBuffer[] crcBufs = new ByteBuffer[dataBufs.length];
-    for(int i = 0; i < crcBufs.length; i++) {
+    final ByteBuffer[] crcBufsC = new ByteBuffer[dataBufs.length];
-      crcBufs[i] = computeCrc(dataBufs[i], bytePerCrc);
+    for(int i = 0; i < dataBufs.length; i++) {
+      crcBufs[i] = computeCrc(dataBufs[i], bytePerCrc,
+          DataChecksum.Type.CRC32);
+      crcBufsC[i] = computeCrc(dataBufs[i], bytePerCrc,
+          DataChecksum.Type.CRC32C);
     }
 
     final String numBytesStr = " bpc ";
     final String numThreadsStr = "#T";
     final String diffStr = "% diff";
 
-    out.print('|');
+    outCrc.print('|');
-    printCell(numBytesStr, 0, out);
+    printCell(numBytesStr, 0, outCrc);
-    printCell(numThreadsStr, 0, out);
+    printCell(numThreadsStr, 0, outCrc);
-    for (int i = 0; i < crcs.size(); i++) {
+    for (int i = 0; i < crcTargets.size(); i++) {
-      final Class<? extends Crc32> c = crcs.get(i);
+      final Class<? extends Crc32> c = crcTargets.get(i);
-      out.print('|');
+      outCrc.print('|');
-      printCell(c.getSimpleName(), 8, out);
+      printCell(c.getSimpleName(), 8, outCrc);
-      for(int j = 0; j < i; j++) {
+      if (i > 0) {
-        printCell(diffStr, diffStr.length(), out);
+        printCell(diffStr, diffStr.length(), outCrc);
       }
     }
-    out.printf("\n");
+    outCrc.printf("\n");
 
     for(int numThreads = 1; numThreads <= dataBufs.length; numThreads <<= 1) {
-      out.printf("|");
+      outCrc.printf("|");
-      printCell(String.valueOf(bytePerCrc), numBytesStr.length(), out);
+      printCell(String.valueOf(bytePerCrc), numBytesStr.length(), outCrc);
-      printCell(String.valueOf(numThreads), numThreadsStr.length(), out);
+      printCell(String.valueOf(numThreads), numThreadsStr.length(), outCrc);
 
       final List<BenchResult> previous = new ArrayList<BenchResult>();
-      for(Class<? extends Crc32> c : crcs) {
+      for(Class<? extends Crc32> c : crcTargets) {
         System.gc();
 
-        final BenchResult result = doBench(c, numThreads, dataBufs, crcBufs,
+        final BenchResult result;
-            bytePerCrc);
+        final Constructor<? extends Crc32> ctor = c.getConstructor();
+        final Crc32 crc = ctor.newInstance();
+        if (crc.crcType() == DataChecksum.Type.CRC32) {
+          result = doBench(c, numThreads, dataBufs, crcBufs, bytePerCrc);
+        } else {
+          result = doBench(c, numThreads, dataBufs, crcBufsC, bytePerCrc);
+        }
         printCell(String.format("%9.1f", result.mbps),
-            c.getSimpleName().length() + 1, out);
+                c.getSimpleName().length() + 1, outCrc);
 
-        //compare result with previous
+        //compare result with the last previous.
-        for(BenchResult p : previous) {
+        final int size = previous.size();
+        if (size > 0) {
+          BenchResult p = previous.get(size - 1);
           final double diff = (result.mbps - p.mbps) / p.mbps * 100;
-          printCell(String.format("%5.1f%%", diff), diffStr.length(), out);
+          printCell(String.format("%5.1f%%", diff), diffStr.length(), outCrc);
         }
         previous.add(result);
       }
-      out.printf("\n");
+      outCrc.printf("\n");
     }
   }
 
-
   private BenchResult doBench(Class<? extends Crc32> clazz,
       final int numThreads, final ByteBuffer[] dataBufs,
       final ByteBuffer[] crcBufs, final int bytePerCrc)
@@ -248,7 +337,6 @@ public class Crc32PerformanceTest {
     final Thread[] threads = new Thread[numThreads];
     final BenchResult[] results = new BenchResult[threads.length];
 
-    {
     final Constructor<? extends Crc32> ctor = clazz.getConstructor();
 
     for(int i = 0; i < threads.length; i++) {
@@ -278,7 +366,6 @@ public class Crc32PerformanceTest {
         }
       };
     }
-    }
 
     for(Thread t : threads) {
       t.start();
@@ -295,7 +382,7 @@ public class Crc32PerformanceTest {
   }
 
   private static class BenchResult {
-    /** Speed (MB per second) */
+    /** Speed (MB per second). */
     final double mbps;
     final Throwable thrown;
 
@@ -321,7 +408,7 @@ public class Crc32PerformanceTest {
     return (System.nanoTime() - startTime) / 1000000000.0d;
   }
 
-  static void printSystemProperties(PrintStream out) {
+  static void printSystemProperties(PrintStream outCrc) {
     final String[] names = {
         "java.version",
         "java.runtime.name",
@@ -344,7 +431,7 @@ public class Crc32PerformanceTest {
 
     final Properties p = System.getProperties();
     for(String n : names) {
-      out.printf("%" + max + "s = %s\n", n, p.getProperty(n));
+      outCrc.printf("%" + max + "s = %s\n", n, p.getProperty(n));
     }
   }
 }