HADOOP-15033. Use java.util.zip.CRC32C for Java 9 and above
Contributed by Dmitry Chuyko,
This commit is contained in:
parent
fbbbf59c82
commit
2e0a451a8d
|
@ -28,6 +28,12 @@ import java.util.zip.Checksum;
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.classification.InterfaceStability;
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
import org.apache.hadoop.fs.ChecksumException;
|
import org.apache.hadoop.fs.ChecksumException;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import java.lang.invoke.MethodHandle;
|
||||||
|
import java.lang.invoke.MethodHandles;
|
||||||
|
import java.lang.invoke.MethodType;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* This class provides interface and utilities for processing checksums for
|
* This class provides interface and utilities for processing checksums for
|
||||||
|
@ -44,6 +50,9 @@ public class DataChecksum implements Checksum {
|
||||||
public static final int CHECKSUM_DEFAULT = 3;
|
public static final int CHECKSUM_DEFAULT = 3;
|
||||||
public static final int CHECKSUM_MIXED = 4;
|
public static final int CHECKSUM_MIXED = 4;
|
||||||
|
|
||||||
|
private static final Logger LOG = LoggerFactory.getLogger(DataChecksum.class);
|
||||||
|
private static volatile boolean useJava9Crc32C = Shell.isJavaVersionAtLeast(9);
|
||||||
|
|
||||||
/** The checksum types */
|
/** The checksum types */
|
||||||
public enum Type {
|
public enum Type {
|
||||||
NULL (CHECKSUM_NULL, 0),
|
NULL (CHECKSUM_NULL, 0),
|
||||||
|
@ -78,6 +87,23 @@ public class DataChecksum implements Checksum {
|
||||||
return new CRC32();
|
return new CRC32();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The flag is volatile to avoid synchronization here.
|
||||||
|
* Re-entrancy is unlikely except in failure mode (and inexpensive).
|
||||||
|
*/
|
||||||
|
static Checksum newCrc32C() {
|
||||||
|
try {
|
||||||
|
return useJava9Crc32C ? Java9Crc32CFactory.createChecksum()
|
||||||
|
: new PureJavaCrc32C();
|
||||||
|
} catch (ExceptionInInitializerError | RuntimeException e) {
|
||||||
|
// should not happen
|
||||||
|
LOG.error("CRC32C creation failed, switching to PureJavaCrc32C", e);
|
||||||
|
useJava9Crc32C = false;
|
||||||
|
return new PureJavaCrc32C();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public static DataChecksum newDataChecksum(Type type, int bytesPerChecksum ) {
|
public static DataChecksum newDataChecksum(Type type, int bytesPerChecksum ) {
|
||||||
if ( bytesPerChecksum <= 0 ) {
|
if ( bytesPerChecksum <= 0 ) {
|
||||||
return null;
|
return null;
|
||||||
|
@ -89,7 +115,7 @@ public class DataChecksum implements Checksum {
|
||||||
case CRC32 :
|
case CRC32 :
|
||||||
return new DataChecksum(type, newCrc32(), bytesPerChecksum );
|
return new DataChecksum(type, newCrc32(), bytesPerChecksum );
|
||||||
case CRC32C:
|
case CRC32C:
|
||||||
return new DataChecksum(type, new PureJavaCrc32C(), bytesPerChecksum);
|
return new DataChecksum(type, newCrc32C(), bytesPerChecksum);
|
||||||
default:
|
default:
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
@ -528,4 +554,36 @@ public class DataChecksum implements Checksum {
|
||||||
@Override
|
@Override
|
||||||
public void update(int b) {}
|
public void update(int b) {}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Holds constructor handle to let it be initialized on demand.
|
||||||
|
*/
|
||||||
|
private static class Java9Crc32CFactory {
|
||||||
|
private static final MethodHandle NEW_CRC32C_MH;
|
||||||
|
|
||||||
|
static {
|
||||||
|
MethodHandle newCRC32C = null;
|
||||||
|
try {
|
||||||
|
newCRC32C = MethodHandles.publicLookup()
|
||||||
|
.findConstructor(
|
||||||
|
Class.forName("java.util.zip.CRC32C"),
|
||||||
|
MethodType.methodType(void.class)
|
||||||
|
);
|
||||||
|
} catch (ReflectiveOperationException e) {
|
||||||
|
// Should not reach here.
|
||||||
|
throw new RuntimeException(e);
|
||||||
|
}
|
||||||
|
NEW_CRC32C_MH = newCRC32C;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Checksum createChecksum() {
|
||||||
|
try {
|
||||||
|
// Should throw nothing
|
||||||
|
return (Checksum) NEW_CRC32C_MH.invoke();
|
||||||
|
} catch (Throwable t) {
|
||||||
|
throw (t instanceof RuntimeException) ? (RuntimeException) t
|
||||||
|
: new RuntimeException(t);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
|
@ -89,6 +89,21 @@ public abstract class Shell {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// "1.8"->8, "9"->9, "10"->10
|
||||||
|
private static final int JAVA_SPEC_VER = Math.max(8, Integer.parseInt(
|
||||||
|
System.getProperty("java.specification.version").split("\\.")[0]));
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Query to see if major version of Java specification of the system
|
||||||
|
* is equal or greater than the parameter.
|
||||||
|
*
|
||||||
|
* @param version 8, 9, 10 etc.
|
||||||
|
* @return comparison with system property, always true for 8
|
||||||
|
*/
|
||||||
|
public static boolean isJavaVersionAtLeast(int version) {
|
||||||
|
return JAVA_SPEC_VER >= version;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Maximum command line length in Windows
|
* Maximum command line length in Windows
|
||||||
* KB830473 documents this as 8191
|
* KB830473 documents this as 8191
|
||||||
|
|
|
@ -129,6 +129,18 @@ public class Crc32PerformanceTest {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
final class ZipC extends AbstractCrc32<Checksum> {
|
||||||
|
@Override
|
||||||
|
public Checksum newAlgorithm() {
|
||||||
|
return DataChecksum.newCrc32C();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public DataChecksum.Type crcType() {
|
||||||
|
return DataChecksum.Type.CRC32C;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
final class PureJava extends AbstractCrc32<PureJavaCrc32> {
|
final class PureJava extends AbstractCrc32<PureJavaCrc32> {
|
||||||
@Override
|
@Override
|
||||||
public PureJavaCrc32 newAlgorithm() {
|
public PureJavaCrc32 newAlgorithm() {
|
||||||
|
@ -169,6 +181,9 @@ public class Crc32PerformanceTest {
|
||||||
this.direct = direct;
|
this.direct = direct;
|
||||||
|
|
||||||
crcs.add(Crc32.Zip.class);
|
crcs.add(Crc32.Zip.class);
|
||||||
|
if (Shell.isJavaVersionAtLeast(9)) {
|
||||||
|
crcs.add(Crc32.ZipC.class);
|
||||||
|
}
|
||||||
crcs.add(Crc32.PureJava.class);
|
crcs.add(Crc32.PureJava.class);
|
||||||
crcs.add(Crc32.PureJavaC.class);
|
crcs.add(Crc32.PureJavaC.class);
|
||||||
|
|
||||||
|
|
|
@ -523,4 +523,9 @@ public class TestShell extends Assert {
|
||||||
shexc1.getProcess().waitFor();
|
shexc1.getProcess().waitFor();
|
||||||
shexc2.getProcess().waitFor();
|
shexc2.getProcess().waitFor();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testIsJavaVersionAtLeast() {
|
||||||
|
assertTrue(Shell.isJavaVersionAtLeast(8));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1571,6 +1571,7 @@
|
||||||
<ignore>sun.nio.ch.*</ignore>
|
<ignore>sun.nio.ch.*</ignore>
|
||||||
<ignore>com.sun.javadoc.*</ignore>
|
<ignore>com.sun.javadoc.*</ignore>
|
||||||
<ignore>com.sun.tools.*</ignore>
|
<ignore>com.sun.tools.*</ignore>
|
||||||
|
<ignore>java.lang.invoke.*</ignore>
|
||||||
</ignores>
|
</ignores>
|
||||||
</configuration>
|
</configuration>
|
||||||
</plugin>
|
</plugin>
|
||||||
|
|
Loading…
Reference in New Issue