HADOOP-15033. Use java.util.zip.CRC32C for Java 9 and above

Contributed by Dmitry Chuyko,

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java

@@ -28,6 +28,12 @@ import java.util.zip.Checksum;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.ChecksumException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.invoke.MethodHandle;
+import java.lang.invoke.MethodHandles;
+import java.lang.invoke.MethodType;
 
 /**
  * This class provides interface and utilities for processing checksums for
@@ -43,6 +49,9 @@ public class DataChecksum implements Checksum {
   public static final int CHECKSUM_CRC32C  = 2;
   public static final int CHECKSUM_DEFAULT = 3; 
   public static final int CHECKSUM_MIXED   = 4;
+
+  private static final Logger LOG = LoggerFactory.getLogger(DataChecksum.class);
+  private static volatile boolean useJava9Crc32C = Shell.isJavaVersionAtLeast(9);
  
   /** The checksum types */
   public enum Type {
@@ -78,6 +87,23 @@ public class DataChecksum implements Checksum {
     return new CRC32();
   }
 
+
+  /**
+   * The flag is volatile to avoid synchronization here.
+   * Re-entrancy is unlikely except in failure mode (and inexpensive).
+   */
+  static Checksum newCrc32C() {
+    try {
+      return useJava9Crc32C ? Java9Crc32CFactory.createChecksum()
+          : new PureJavaCrc32C();
+    } catch (ExceptionInInitializerError | RuntimeException e) {
+      // should not happen
+      LOG.error("CRC32C creation failed, switching to PureJavaCrc32C", e);
+      useJava9Crc32C = false;
+      return new PureJavaCrc32C();
+    }
+  }
+
   public static DataChecksum newDataChecksum(Type type, int bytesPerChecksum ) {
     if ( bytesPerChecksum <= 0 ) {
       return null;
@@ -89,7 +115,7 @@ public class DataChecksum implements Checksum {
     case CRC32 :
       return new DataChecksum(type, newCrc32(), bytesPerChecksum );
     case CRC32C:
-      return new DataChecksum(type, new PureJavaCrc32C(), bytesPerChecksum);
+      return new DataChecksum(type, newCrc32C(), bytesPerChecksum);
     default:
       return null;  
     }
@@ -528,4 +554,36 @@ public class DataChecksum implements Checksum {
     @Override
     public void update(int b) {}
   };
+
+  /**
+   * Holds constructor handle to let it be initialized on demand.
+   */
+  private static class Java9Crc32CFactory {
+    private static final MethodHandle NEW_CRC32C_MH;
+
+    static {
+      MethodHandle newCRC32C = null;
+      try {
+        newCRC32C = MethodHandles.publicLookup()
+            .findConstructor(
+                Class.forName("java.util.zip.CRC32C"),
+                MethodType.methodType(void.class)
+            );
+      } catch (ReflectiveOperationException e) {
+        // Should not reach here.
+        throw new RuntimeException(e);
+      }
+      NEW_CRC32C_MH = newCRC32C;
+    }
+
+    public static Checksum createChecksum() {
+      try {
+        // Should throw nothing
+        return (Checksum) NEW_CRC32C_MH.invoke();
+      } catch (Throwable t) {
+        throw (t instanceof RuntimeException) ? (RuntimeException) t
+            : new RuntimeException(t);
+      }
+    }
+  };
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java

@@ -89,6 +89,21 @@ public abstract class Shell {
     return true;
   }
 
+  // "1.8"->8, "9"->9, "10"->10
+  private static final int JAVA_SPEC_VER = Math.max(8, Integer.parseInt(
+      System.getProperty("java.specification.version").split("\\.")[0]));
+
+  /**
+   * Query to see if major version of Java specification of the system
+   * is equal or greater than the parameter.
+   *
+   * @param version 8, 9, 10 etc.
+   * @return comparison with system property, always true for 8
+   */
+  public static boolean isJavaVersionAtLeast(int version) {
+    return JAVA_SPEC_VER >= version;
+  }
+
   /**
    * Maximum command line length in Windows
    * KB830473 documents this as 8191

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/Crc32PerformanceTest.java

@@ -129,6 +129,18 @@ public class Crc32PerformanceTest {
       }
     }
 
+    final class ZipC extends AbstractCrc32<Checksum> {
+      @Override
+      public Checksum newAlgorithm() {
+        return DataChecksum.newCrc32C();
+      }
+
+      @Override
+      public DataChecksum.Type crcType() {
+        return DataChecksum.Type.CRC32C;
+      }
+    }
+
     final class PureJava extends AbstractCrc32<PureJavaCrc32> {
       @Override
       public PureJavaCrc32 newAlgorithm() {
@@ -169,6 +181,9 @@ public class Crc32PerformanceTest {
     this.direct = direct;
 
     crcs.add(Crc32.Zip.class);
+    if (Shell.isJavaVersionAtLeast(9)) {
+      crcs.add(Crc32.ZipC.class);
+    }
     crcs.add(Crc32.PureJava.class);
     crcs.add(Crc32.PureJavaC.class);
 
@@ -435,4 +450,4 @@ public class Crc32PerformanceTest {
       outCrc.printf("%" + max + "s = %s\n", n, p.getProperty(n));
     }
   }
-}
+}

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java

@@ -523,4 +523,9 @@ public class TestShell extends Assert {
     shexc1.getProcess().waitFor();
     shexc2.getProcess().waitFor();
   }
+
+  @Test
+  public void testIsJavaVersionAtLeast() {
+    assertTrue(Shell.isJavaVersionAtLeast(8));
+  }
 }

hadoop-project/pom.xml

@@ -1571,6 +1571,7 @@
             <ignore>sun.nio.ch.*</ignore>
             <ignore>com.sun.javadoc.*</ignore>
             <ignore>com.sun.tools.*</ignore>
+            <ignore>java.lang.invoke.*</ignore>
           </ignores>
         </configuration>
       </plugin>