From ddaeb3e4979338551573285f219a2d361f502950 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Thu, 27 Apr 2017 12:25:47 -0700 Subject: [PATCH] Revert "HADOOP-13200. Implement customizable and configurable erasure coders. Contributed by Tim Yao." This reverts commit 872088c6e7e78534a8ffd1ea6e57b86faea3d6e1. --- .../hadoop/io/erasurecode/CodecUtil.java | 82 +++++++++++++------ .../io/erasurecode/ErasureCodeConstants.java | 1 - .../rawcoder/DummyRawErasureCoderFactory.java | 12 --- .../NativeRSRawErasureCoderFactory.java | 13 --- .../NativeXORRawErasureCoderFactory.java | 13 --- .../RSLegacyRawErasureCoderFactory.java | 13 --- .../rawcoder/RSRawErasureCoderFactory.java | 13 --- .../rawcoder/RawErasureCoderFactory.java | 12 --- .../rawcoder/XORRawErasureCoderFactory.java | 13 --- .../src/main/resources/core-default.xml | 6 +- .../erasurecode/TestCodecRawCoderMapping.java | 8 +- .../coder/TestHHXORErasureCoder.java | 2 +- .../erasurecode/coder/TestRSErasureCoder.java | 2 +- .../src/site/markdown/HDFSErasureCoding.md | 14 ++-- .../hdfs/TestDFSStripedInputStream.java | 2 +- .../hdfs/TestDFSStripedOutputStream.java | 2 +- ...TestDFSStripedOutputStreamWithFailure.java | 2 +- .../hdfs/TestReconstructStripedFile.java | 2 +- .../TestUnsetAndChangeDirectoryEcPolicy.java | 2 +- 19 files changed, 76 insertions(+), 138 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java index 6cdbb37a022..c8b6a6839cc 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java @@ -18,6 +18,8 @@ package org.apache.hadoop.io.erasurecode; import com.google.common.base.Preconditions; +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -28,12 +30,18 @@ import org.apache.hadoop.io.erasurecode.codec.RSErasureCodec; import org.apache.hadoop.io.erasurecode.codec.XORErasureCodec; import org.apache.hadoop.io.erasurecode.coder.ErasureDecoder; import org.apache.hadoop.io.erasurecode.coder.ErasureEncoder; +import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory; +import org.apache.hadoop.io.erasurecode.rawcoder.NativeXORRawErasureCoderFactory; +import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory; +import org.apache.hadoop.io.erasurecode.rawcoder.RSLegacyRawErasureCoderFactory; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureCoderFactory; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder; +import org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; +import java.util.Map; /** * A codec & coder utility to help create coders conveniently. @@ -71,12 +79,27 @@ public final class CodecUtil { /** Comma separated raw codec name. The first coder is prior to the latter. */ public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODERS_KEY = IO_ERASURECODE_CODEC + "rs-legacy.rawcoders"; + public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODERS_DEFAULT = + RSLegacyRawErasureCoderFactory.class.getCanonicalName(); public static final String IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY = IO_ERASURECODE_CODEC + "rs.rawcoders"; + public static final String IO_ERASURECODE_CODEC_RS_RAWCODERS_DEFAULT = + NativeRSRawErasureCoderFactory.class.getCanonicalName() + + "," + RSRawErasureCoderFactory.class.getCanonicalName(); /** Raw coder factory for the XOR codec. */ public static final String IO_ERASURECODE_CODEC_XOR_RAWCODERS_KEY = IO_ERASURECODE_CODEC + "xor.rawcoders"; + public static final String IO_ERASURECODE_CODEC_XOR_RAWCODERS_DEFAULT = + NativeXORRawErasureCoderFactory.class.getCanonicalName() + + "," + XORRawErasureCoderFactory.class.getCanonicalName(); + + // Default coders for each codec names. + public static final Map DEFAULT_CODERS_MAP = ImmutableMap.of( + "rs", IO_ERASURECODE_CODEC_RS_RAWCODERS_DEFAULT, + "rs-legacy", IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODERS_DEFAULT, + "xor", IO_ERASURECODE_CODEC_XOR_RAWCODERS_DEFAULT + ); private CodecUtil() { } @@ -145,61 +168,70 @@ public final class CodecUtil { } private static RawErasureCoderFactory createRawCoderFactory( - String coderName, String codecName) { + Configuration conf, String rawCoderFactoryKey) { RawErasureCoderFactory fact; - fact = CodecRegistry.getInstance(). - getCoderByName(codecName, coderName); + try { + Class factClass = conf.getClassByName( + rawCoderFactoryKey).asSubclass(RawErasureCoderFactory.class); + fact = factClass.newInstance(); + } catch (ClassNotFoundException | InstantiationException | + IllegalAccessException e) { + throw new RuntimeException("Failed to create raw coder factory", e); + } + + if (fact == null) { + throw new RuntimeException("Failed to create raw coder factory"); + } return fact; } - // Return a list of coder names - private static String[] getRawCoderNames( - Configuration conf, String codecName) { - return conf.getStrings( - IO_ERASURECODE_CODEC + codecName + ".rawcoders", - CodecRegistry.getInstance().getCoderNames(codecName) + // Return comma separated coder names + private static String getRawCoders(Configuration conf, String codec) { + return conf.get( + IO_ERASURECODE_CODEC + codec + ".rawcoders", + DEFAULT_CODERS_MAP.getOrDefault(codec, codec) ); } private static RawErasureEncoder createRawEncoderWithFallback( - Configuration conf, String codecName, ErasureCoderOptions coderOptions) { - String[] rawCoderNames = getRawCoderNames(conf, codecName); - for (String rawCoderName : rawCoderNames) { + Configuration conf, String codec, ErasureCoderOptions coderOptions) { + String coders = getRawCoders(conf, codec); + for (String factName : Splitter.on(",").split(coders)) { try { - if (rawCoderName != null) { - RawErasureCoderFactory fact = createRawCoderFactory( - rawCoderName, codecName); + if (factName != null) { + RawErasureCoderFactory fact = createRawCoderFactory(conf, + factName); return fact.createEncoder(coderOptions); } } catch (LinkageError | Exception e) { // Fallback to next coder if possible - LOG.warn("Failed to create raw erasure encoder " + rawCoderName + + LOG.warn("Failed to create raw erasure encoder " + factName + ", fallback to next codec if possible", e); } } throw new IllegalArgumentException("Fail to create raw erasure " + - "encoder with given codec: " + codecName); + "encoder with given codec: " + codec); } private static RawErasureDecoder createRawDecoderWithFallback( - Configuration conf, String codecName, ErasureCoderOptions coderOptions) { - String[] coders = getRawCoderNames(conf, codecName); - for (String rawCoderName : coders) { + Configuration conf, String codec, ErasureCoderOptions coderOptions) { + String coders = getRawCoders(conf, codec); + for (String factName : Splitter.on(",").split(coders)) { try { - if (rawCoderName != null) { - RawErasureCoderFactory fact = createRawCoderFactory( - rawCoderName, codecName); + if (factName != null) { + RawErasureCoderFactory fact = createRawCoderFactory(conf, + factName); return fact.createDecoder(coderOptions); } } catch (LinkageError | Exception e) { // Fallback to next coder if possible - LOG.warn("Failed to create raw erasure decoder " + rawCoderName + + LOG.warn("Failed to create raw erasure decoder " + factName + ", fallback to next codec if possible", e); } } throw new IllegalArgumentException("Fail to create raw erasure " + - "encoder with given codec: " + codecName); + "encoder with given codec: " + codec); } private static ErasureCodec createCodec(Configuration conf, diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java index e0d7946e7f7..c830bb2b6f7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java @@ -25,7 +25,6 @@ public final class ErasureCodeConstants { private ErasureCodeConstants() { } - public static final String DUMMY_CODEC_NAME = "dummy"; public static final String RS_CODEC_NAME = "rs"; public static final String RS_LEGACY_CODEC_NAME = "rs-legacy"; public static final String XOR_CODEC_NAME = "xor"; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/DummyRawErasureCoderFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/DummyRawErasureCoderFactory.java index 49b36b7561f..31ba4efc04f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/DummyRawErasureCoderFactory.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/DummyRawErasureCoderFactory.java @@ -18,7 +18,6 @@ package org.apache.hadoop.io.erasurecode.rawcoder; import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; /** @@ -26,7 +25,6 @@ import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; */ @InterfaceAudience.Private public class DummyRawErasureCoderFactory implements RawErasureCoderFactory { - public static final String CODER_NAME = "dummy_dummy"; @Override public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) { @@ -37,14 +35,4 @@ public class DummyRawErasureCoderFactory implements RawErasureCoderFactory { public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) { return new DummyRawDecoder(coderOptions); } - - @Override - public String getCoderName() { - return CODER_NAME; - } - - @Override - public String getCodecName() { - return ErasureCodeConstants.DUMMY_CODEC_NAME; - } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawErasureCoderFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawErasureCoderFactory.java index cc98ec6ccb7..38997dd301e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawErasureCoderFactory.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawErasureCoderFactory.java @@ -18,7 +18,6 @@ package org.apache.hadoop.io.erasurecode.rawcoder; import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; /** @@ -28,8 +27,6 @@ import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; @InterfaceAudience.Private public class NativeRSRawErasureCoderFactory implements RawErasureCoderFactory { - public static final String CODER_NAME = "rs_native"; - @Override public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) { return new NativeRSRawEncoder(coderOptions); @@ -39,14 +36,4 @@ public class NativeRSRawErasureCoderFactory implements RawErasureCoderFactory { public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) { return new NativeRSRawDecoder(coderOptions); } - - @Override - public String getCoderName() { - return CODER_NAME; - } - - @Override - public String getCodecName() { - return ErasureCodeConstants.RS_CODEC_NAME; - } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawErasureCoderFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawErasureCoderFactory.java index 29a00dc1da2..66b3f7830a9 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawErasureCoderFactory.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawErasureCoderFactory.java @@ -18,7 +18,6 @@ package org.apache.hadoop.io.erasurecode.rawcoder; import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; /** @@ -28,8 +27,6 @@ import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; @InterfaceAudience.Private public class NativeXORRawErasureCoderFactory implements RawErasureCoderFactory { - public static final String CODER_NAME = "xor_native"; - @Override public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) { return new NativeXORRawEncoder(coderOptions); @@ -39,14 +36,4 @@ public class NativeXORRawErasureCoderFactory implements RawErasureCoderFactory { public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) { return new NativeXORRawDecoder(coderOptions); } - - @Override - public String getCoderName() { - return CODER_NAME; - } - - @Override - public String getCodecName() { - return ErasureCodeConstants.XOR_CODEC_NAME; - } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSLegacyRawErasureCoderFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSLegacyRawErasureCoderFactory.java index 6cbe33f6b9a..45bbf40f690 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSLegacyRawErasureCoderFactory.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSLegacyRawErasureCoderFactory.java @@ -18,7 +18,6 @@ package org.apache.hadoop.io.erasurecode.rawcoder; import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; /** @@ -27,8 +26,6 @@ import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; @InterfaceAudience.Private public class RSLegacyRawErasureCoderFactory implements RawErasureCoderFactory { - public static final String CODER_NAME = "rs-legacy_java"; - @Override public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) { return new RSLegacyRawEncoder(coderOptions); @@ -38,14 +35,4 @@ public class RSLegacyRawErasureCoderFactory implements RawErasureCoderFactory { public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) { return new RSLegacyRawDecoder(coderOptions); } - - @Override - public String getCoderName() { - return CODER_NAME; - } - - @Override - public String getCodecName() { - return ErasureCodeConstants.RS_LEGACY_CODEC_NAME; - } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawErasureCoderFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawErasureCoderFactory.java index 4aadcb72116..8d954d59ca8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawErasureCoderFactory.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawErasureCoderFactory.java @@ -18,7 +18,6 @@ package org.apache.hadoop.io.erasurecode.rawcoder; import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; /** @@ -27,8 +26,6 @@ import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; @InterfaceAudience.Private public class RSRawErasureCoderFactory implements RawErasureCoderFactory { - public static final String CODER_NAME = "rs_java"; - @Override public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) { return new RSRawEncoder(coderOptions); @@ -38,14 +35,4 @@ public class RSRawErasureCoderFactory implements RawErasureCoderFactory { public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) { return new RSRawDecoder(coderOptions); } - - @Override - public String getCoderName() { - return CODER_NAME; - } - - @Override - public String getCodecName() { - return ErasureCodeConstants.RS_CODEC_NAME; - } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderFactory.java index 704e3e8db68..05f228b3037 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderFactory.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderFactory.java @@ -41,16 +41,4 @@ public interface RawErasureCoderFactory { * @return raw erasure decoder */ RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions); - - /** - * Get the name of the coder. - * @return coder name - */ - String getCoderName(); - - /** - * Get the name of its codec. - * @return codec name - */ - String getCodecName(); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawErasureCoderFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawErasureCoderFactory.java index 595f022e7bd..571fe12245c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawErasureCoderFactory.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawErasureCoderFactory.java @@ -18,7 +18,6 @@ package org.apache.hadoop.io.erasurecode.rawcoder; import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; /** @@ -27,8 +26,6 @@ import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; @InterfaceAudience.Private public class XORRawErasureCoderFactory implements RawErasureCoderFactory { - public static final String CODER_NAME = "xor_java"; - @Override public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) { return new XORRawEncoder(coderOptions); @@ -38,14 +35,4 @@ public class XORRawErasureCoderFactory implements RawErasureCoderFactory { public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) { return new XORRawDecoder(coderOptions); } - - @Override - public String getCoderName() { - return CODER_NAME; - } - - @Override - public String getCodecName() { - return ErasureCodeConstants.XOR_CODEC_NAME; - } } diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index e9a4f1a2d2d..6fa70fd0d4f 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -668,7 +668,7 @@ io.erasurecode.codec.rs.rawcoders - rs_native,rs_java + org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory,org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory Comma separated raw coder implementations for the rs codec. The earlier factory is prior to followings in case of failure of creating raw coders. @@ -677,7 +677,7 @@ io.erasurecode.codec.rs-legacy.rawcoders - rs-legacy_java + org.apache.hadoop.io.erasurecode.rawcoder.RSLegacyRawErasureCoderFactory Comma separated raw coder implementations for the rs-legacy codec. The earlier factory is prior to followings in case of failure of creating raw coders. @@ -686,7 +686,7 @@ io.erasurecode.codec.xor.rawcoders - xor_native,xor_java + org.apache.hadoop.io.erasurecode.rawcoder.NativeXORRawErasureCoderFactory,org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory Comma separated raw coder implementations for the xor codec. The earlier factory is prior to followings in case of failure of creating raw coders. diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java index 4a6fbd41f55..b5cada65fa2 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java @@ -28,7 +28,6 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder; import org.apache.hadoop.io.erasurecode.rawcoder.XORRawDecoder; import org.apache.hadoop.io.erasurecode.rawcoder.XORRawEncoder; -import org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory; import org.apache.hadoop.test.GenericTestUtils; import org.junit.Assert; import org.junit.Before; @@ -105,8 +104,8 @@ public class TestCodecRawCoderMapping { ErasureCoderOptions coderOptions = new ErasureCoderOptions( numDataUnit, numParityUnit); conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY, - RSRawErasureCoderFactory.CODER_NAME + - "," + NativeRSRawErasureCoderFactory.CODER_NAME); + RSRawErasureCoderFactory.class.getCanonicalName() + + "," + NativeRSRawErasureCoderFactory.class.getCanonicalName()); // should return default raw coder of rs codec RawErasureEncoder encoder = CodecUtil.createRawEncoder( conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions); @@ -134,7 +133,8 @@ public class TestCodecRawCoderMapping { ErasureCoderOptions coderOptions = new ErasureCoderOptions( numDataUnit, numParityUnit); conf.set(CodecUtil.IO_ERASURECODE_CODEC_XOR_RAWCODERS_KEY, - "invalid-codec," + XORRawErasureCoderFactory.CODER_NAME); + "invalid-codec," + + "org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory"); // should return second coder specified by IO_ERASURECODE_CODEC_CODERS RawErasureEncoder encoder = CodecUtil.createRawEncoder( conf, ErasureCodeConstants.XOR_CODEC_NAME, coderOptions); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java index 094ed0801e0..a475a39f678 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java @@ -51,7 +51,7 @@ public class TestHHXORErasureCoder extends TestHHErasureCoderBase { */ Configuration conf = new Configuration(); conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY, - RSRawErasureCoderFactory.CODER_NAME); + RSRawErasureCoderFactory.class.getCanonicalName()); prepare(conf, 10, 4, new int[]{0}, new int[0]); testCoding(true); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java index 19054cecff0..11abac40b82 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java @@ -58,7 +58,7 @@ public class TestRSErasureCoder extends TestErasureCoderBase { */ Configuration conf = new Configuration(); conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY, - RSRawErasureCoderFactory.CODER_NAME); + RSRawErasureCoderFactory.class.getCanonicalName()); prepare(conf, 10, 4, new int[]{0}, new int[0]); testCoding(true); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md index dbfb1118b37..543b5ceb94e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md @@ -117,15 +117,11 @@ Deployment be more appropriate. If the administrator only cares about node-level fault-tolerance, `RS-10-4-64k` would still be appropriate as long as there are at least 14 DataNodes in the cluster. - The codec implementations for Reed-Solomon and XOR can be configured with the following client and DataNode configuration keys: - `io.erasurecode.codec.rs.rawcoders` for the default RS codec, - `io.erasurecode.codec.rs-legacy.rawcoders` for the legacy RS codec, - `io.erasurecode.codec.xor.rawcoders` for the XOR codec. - User can also configure self-defined codec with configuration key like: - `io.erasurecode.codec.self-defined-codec.rawcoders`. - The values for these key are lists of coder names with a fall-back mechanism. - All these codecs have implementations in pure Java. For default RS codec, there is also a native implementation which leverages Intel ISA-L library to improve the performance of codec. For XOR codec, a native implementation which leverages Intel ISA-L library to improve the performance of codec is also supported. Please refer to section "Enable Intel ISA-L" for more detail information. - The default implementation for RS Legacy is pure Java, and the default implementations for default RS and XOR are native implementations using Intel ISA-L library. + The codec implementation for Reed-Solomon and XOR can be configured with the following client and DataNode configuration keys: + `io.erasurecode.codec.rs.rawcoder` for the default RS codec, + `io.erasurecode.codec.rs-legacy.rawcoder` for the legacy RS codec, + `io.erasurecode.codec.xor.rawcoder` for the XOR codec. + The default implementations for all of these codecs are pure Java. For default RS codec, there is also a native implementation which leverages Intel ISA-L library to improve the performance of codec. For XOR codec, a native implementation which leverages Intel ISA-L library to improve the performance of codec is also supported. Please refer to section "Enable Intel ISA-L" for more detail information. Erasure coding background recovery work on the DataNodes can also be tuned via the following configuration parameters: diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java index 4f67a0a1a29..e3332fd403e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java @@ -98,7 +98,7 @@ public class TestDFSStripedInputStream { if (ErasureCodeNative.isNativeCodeLoaded()) { conf.set( CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY, - NativeRSRawErasureCoderFactory.CODER_NAME); + NativeRSRawErasureCoderFactory.class.getCanonicalName()); } SimulatedFSDataset.setFactory(conf); cluster = new MiniDFSCluster.Builder(conf).numDataNodes( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java index 14825ca41b8..a2d5c8d7b06 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java @@ -85,7 +85,7 @@ public class TestDFSStripedOutputStream { if (ErasureCodeNative.isNativeCodeLoaded()) { conf.set( CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY, - NativeRSRawErasureCoderFactory.CODER_NAME); + NativeRSRawErasureCoderFactory.class.getCanonicalName()); } DFSTestUtil.enableAllECPolicies(conf); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java index 9915a2fb6d9..66a805c43fd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java @@ -214,7 +214,7 @@ public class TestDFSStripedOutputStreamWithFailure { if (ErasureCodeNative.isNativeCodeLoaded()) { conf.set( CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY, - NativeRSRawErasureCoderFactory.CODER_NAME); + NativeRSRawErasureCoderFactory.class.getCanonicalName()); } DFSTestUtil.enableAllECPolicies(conf); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java index 29e4028f559..2bd4a905bce 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java @@ -100,7 +100,7 @@ public class TestReconstructStripedFile { if (ErasureCodeNative.isNativeCodeLoaded()) { conf.set( CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY, - NativeRSRawErasureCoderFactory.CODER_NAME); + NativeRSRawErasureCoderFactory.class.getCanonicalName()); } conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, StripedFileTestUtil.getDefaultECPolicy().getName()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java index 5371e205ac3..6379db5ed1c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java @@ -68,7 +68,7 @@ public class TestUnsetAndChangeDirectoryEcPolicy { if (ErasureCodeNative.isNativeCodeLoaded()) { conf.set( CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY, - NativeRSRawErasureCoderFactory.CODER_NAME); + NativeRSRawErasureCoderFactory.class.getCanonicalName()); } DFSTestUtil.enableAllECPolicies(conf); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(