diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java index 03946afe04d..43e377f1fb9 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java @@ -28,6 +28,12 @@ import java.util.zip.Checksum; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.ChecksumException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; /** * This class provides interface and utilities for processing checksums for @@ -43,6 +49,9 @@ public class DataChecksum implements Checksum { public static final int CHECKSUM_CRC32C = 2; public static final int CHECKSUM_DEFAULT = 3; public static final int CHECKSUM_MIXED = 4; + + private static final Logger LOG = LoggerFactory.getLogger(DataChecksum.class); + private static volatile boolean useJava9Crc32C = Shell.isJavaVersionAtLeast(9); /** The checksum types */ public enum Type { @@ -78,6 +87,23 @@ public class DataChecksum implements Checksum { return new CRC32(); } + + /** + * The flag is volatile to avoid synchronization here. + * Re-entrancy is unlikely except in failure mode (and inexpensive). + */ + static Checksum newCrc32C() { + try { + return useJava9Crc32C ? Java9Crc32CFactory.createChecksum() + : new PureJavaCrc32C(); + } catch (ExceptionInInitializerError | RuntimeException e) { + // should not happen + LOG.error("CRC32C creation failed, switching to PureJavaCrc32C", e); + useJava9Crc32C = false; + return new PureJavaCrc32C(); + } + } + public static DataChecksum newDataChecksum(Type type, int bytesPerChecksum ) { if ( bytesPerChecksum <= 0 ) { return null; @@ -89,7 +115,7 @@ public class DataChecksum implements Checksum { case CRC32 : return new DataChecksum(type, newCrc32(), bytesPerChecksum ); case CRC32C: - return new DataChecksum(type, new PureJavaCrc32C(), bytesPerChecksum); + return new DataChecksum(type, newCrc32C(), bytesPerChecksum); default: return null; } @@ -528,4 +554,36 @@ public class DataChecksum implements Checksum { @Override public void update(int b) {} }; + + /** + * Holds constructor handle to let it be initialized on demand. + */ + private static class Java9Crc32CFactory { + private static final MethodHandle NEW_CRC32C_MH; + + static { + MethodHandle newCRC32C = null; + try { + newCRC32C = MethodHandles.publicLookup() + .findConstructor( + Class.forName("java.util.zip.CRC32C"), + MethodType.methodType(void.class) + ); + } catch (ReflectiveOperationException e) { + // Should not reach here. + throw new RuntimeException(e); + } + NEW_CRC32C_MH = newCRC32C; + } + + public static Checksum createChecksum() { + try { + // Should throw nothing + return (Checksum) NEW_CRC32C_MH.invoke(); + } catch (Throwable t) { + throw (t instanceof RuntimeException) ? (RuntimeException) t + : new RuntimeException(t); + } + } + }; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java index bfb8183be25..c25cba24793 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java @@ -89,6 +89,21 @@ public abstract class Shell { return true; } + // "1.8"->8, "9"->9, "10"->10 + private static final int JAVA_SPEC_VER = Math.max(8, Integer.parseInt( + System.getProperty("java.specification.version").split("\\.")[0])); + + /** + * Query to see if major version of Java specification of the system + * is equal or greater than the parameter. + * + * @param version 8, 9, 10 etc. + * @return comparison with system property, always true for 8 + */ + public static boolean isJavaVersionAtLeast(int version) { + return JAVA_SPEC_VER >= version; + } + /** * Maximum command line length in Windows * KB830473 documents this as 8191 diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/Crc32PerformanceTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/Crc32PerformanceTest.java index ce28f503a84..f0d6145b9d7 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/Crc32PerformanceTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/Crc32PerformanceTest.java @@ -129,6 +129,18 @@ public class Crc32PerformanceTest { } } + final class ZipC extends AbstractCrc32 { + @Override + public Checksum newAlgorithm() { + return DataChecksum.newCrc32C(); + } + + @Override + public DataChecksum.Type crcType() { + return DataChecksum.Type.CRC32C; + } + } + final class PureJava extends AbstractCrc32 { @Override public PureJavaCrc32 newAlgorithm() { @@ -169,6 +181,9 @@ public class Crc32PerformanceTest { this.direct = direct; crcs.add(Crc32.Zip.class); + if (Shell.isJavaVersionAtLeast(9)) { + crcs.add(Crc32.ZipC.class); + } crcs.add(Crc32.PureJava.class); crcs.add(Crc32.PureJavaC.class); @@ -435,4 +450,4 @@ public class Crc32PerformanceTest { outCrc.printf("%" + max + "s = %s\n", n, p.getProperty(n)); } } -} \ No newline at end of file +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java index 4f30edb0056..d0ebc2b83f7 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java @@ -523,4 +523,9 @@ public class TestShell extends Assert { shexc1.getProcess().waitFor(); shexc2.getProcess().waitFor(); } + + @Test + public void testIsJavaVersionAtLeast() { + assertTrue(Shell.isJavaVersionAtLeast(8)); + } } diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 3c49182b2f0..a73b6af71b3 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -1571,6 +1571,7 @@ sun.nio.ch.* com.sun.javadoc.* com.sun.tools.* + java.lang.invoke.*