Revert "HADOOP-13200. Implement customizable and configurable erasure coders. Contributed by Tim Yao."

This reverts commit 872088c6e7.
This commit is contained in:
Wei-Chiu Chuang 2017-04-27 12:25:47 -07:00
parent 872088c6e7
commit ddaeb3e497
19 changed files with 76 additions and 138 deletions

View File

@ -18,6 +18,8 @@
package org.apache.hadoop.io.erasurecode; package org.apache.hadoop.io.erasurecode;
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableMap;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
@ -28,12 +30,18 @@ import org.apache.hadoop.io.erasurecode.codec.RSErasureCodec;
import org.apache.hadoop.io.erasurecode.codec.XORErasureCodec; import org.apache.hadoop.io.erasurecode.codec.XORErasureCodec;
import org.apache.hadoop.io.erasurecode.coder.ErasureDecoder; import org.apache.hadoop.io.erasurecode.coder.ErasureDecoder;
import org.apache.hadoop.io.erasurecode.coder.ErasureEncoder; import org.apache.hadoop.io.erasurecode.coder.ErasureEncoder;
import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory;
import org.apache.hadoop.io.erasurecode.rawcoder.NativeXORRawErasureCoderFactory;
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory;
import org.apache.hadoop.io.erasurecode.rawcoder.RSLegacyRawErasureCoderFactory;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureCoderFactory; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureCoderFactory;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
import org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory;
import java.lang.reflect.Constructor; import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException; import java.lang.reflect.InvocationTargetException;
import java.util.Map;
/** /**
* A codec & coder utility to help create coders conveniently. * A codec & coder utility to help create coders conveniently.
@ -71,12 +79,27 @@ public final class CodecUtil {
/** Comma separated raw codec name. The first coder is prior to the latter. */ /** Comma separated raw codec name. The first coder is prior to the latter. */
public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODERS_KEY = public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODERS_KEY =
IO_ERASURECODE_CODEC + "rs-legacy.rawcoders"; IO_ERASURECODE_CODEC + "rs-legacy.rawcoders";
public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODERS_DEFAULT =
RSLegacyRawErasureCoderFactory.class.getCanonicalName();
public static final String IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY = public static final String IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY =
IO_ERASURECODE_CODEC + "rs.rawcoders"; IO_ERASURECODE_CODEC + "rs.rawcoders";
public static final String IO_ERASURECODE_CODEC_RS_RAWCODERS_DEFAULT =
NativeRSRawErasureCoderFactory.class.getCanonicalName() +
"," + RSRawErasureCoderFactory.class.getCanonicalName();
/** Raw coder factory for the XOR codec. */ /** Raw coder factory for the XOR codec. */
public static final String IO_ERASURECODE_CODEC_XOR_RAWCODERS_KEY = public static final String IO_ERASURECODE_CODEC_XOR_RAWCODERS_KEY =
IO_ERASURECODE_CODEC + "xor.rawcoders"; IO_ERASURECODE_CODEC + "xor.rawcoders";
public static final String IO_ERASURECODE_CODEC_XOR_RAWCODERS_DEFAULT =
NativeXORRawErasureCoderFactory.class.getCanonicalName() +
"," + XORRawErasureCoderFactory.class.getCanonicalName();
// Default coders for each codec names.
public static final Map<String, String> DEFAULT_CODERS_MAP = ImmutableMap.of(
"rs", IO_ERASURECODE_CODEC_RS_RAWCODERS_DEFAULT,
"rs-legacy", IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODERS_DEFAULT,
"xor", IO_ERASURECODE_CODEC_XOR_RAWCODERS_DEFAULT
);
private CodecUtil() { } private CodecUtil() { }
@ -145,61 +168,70 @@ public final class CodecUtil {
} }
private static RawErasureCoderFactory createRawCoderFactory( private static RawErasureCoderFactory createRawCoderFactory(
String coderName, String codecName) { Configuration conf, String rawCoderFactoryKey) {
RawErasureCoderFactory fact; RawErasureCoderFactory fact;
fact = CodecRegistry.getInstance(). try {
getCoderByName(codecName, coderName); Class<? extends RawErasureCoderFactory> factClass = conf.getClassByName(
rawCoderFactoryKey).asSubclass(RawErasureCoderFactory.class);
fact = factClass.newInstance();
} catch (ClassNotFoundException | InstantiationException |
IllegalAccessException e) {
throw new RuntimeException("Failed to create raw coder factory", e);
}
if (fact == null) {
throw new RuntimeException("Failed to create raw coder factory");
}
return fact; return fact;
} }
// Return a list of coder names // Return comma separated coder names
private static String[] getRawCoderNames( private static String getRawCoders(Configuration conf, String codec) {
Configuration conf, String codecName) { return conf.get(
return conf.getStrings( IO_ERASURECODE_CODEC + codec + ".rawcoders",
IO_ERASURECODE_CODEC + codecName + ".rawcoders", DEFAULT_CODERS_MAP.getOrDefault(codec, codec)
CodecRegistry.getInstance().getCoderNames(codecName)
); );
} }
private static RawErasureEncoder createRawEncoderWithFallback( private static RawErasureEncoder createRawEncoderWithFallback(
Configuration conf, String codecName, ErasureCoderOptions coderOptions) { Configuration conf, String codec, ErasureCoderOptions coderOptions) {
String[] rawCoderNames = getRawCoderNames(conf, codecName); String coders = getRawCoders(conf, codec);
for (String rawCoderName : rawCoderNames) { for (String factName : Splitter.on(",").split(coders)) {
try { try {
if (rawCoderName != null) { if (factName != null) {
RawErasureCoderFactory fact = createRawCoderFactory( RawErasureCoderFactory fact = createRawCoderFactory(conf,
rawCoderName, codecName); factName);
return fact.createEncoder(coderOptions); return fact.createEncoder(coderOptions);
} }
} catch (LinkageError | Exception e) { } catch (LinkageError | Exception e) {
// Fallback to next coder if possible // Fallback to next coder if possible
LOG.warn("Failed to create raw erasure encoder " + rawCoderName + LOG.warn("Failed to create raw erasure encoder " + factName +
", fallback to next codec if possible", e); ", fallback to next codec if possible", e);
} }
} }
throw new IllegalArgumentException("Fail to create raw erasure " + throw new IllegalArgumentException("Fail to create raw erasure " +
"encoder with given codec: " + codecName); "encoder with given codec: " + codec);
} }
private static RawErasureDecoder createRawDecoderWithFallback( private static RawErasureDecoder createRawDecoderWithFallback(
Configuration conf, String codecName, ErasureCoderOptions coderOptions) { Configuration conf, String codec, ErasureCoderOptions coderOptions) {
String[] coders = getRawCoderNames(conf, codecName); String coders = getRawCoders(conf, codec);
for (String rawCoderName : coders) { for (String factName : Splitter.on(",").split(coders)) {
try { try {
if (rawCoderName != null) { if (factName != null) {
RawErasureCoderFactory fact = createRawCoderFactory( RawErasureCoderFactory fact = createRawCoderFactory(conf,
rawCoderName, codecName); factName);
return fact.createDecoder(coderOptions); return fact.createDecoder(coderOptions);
} }
} catch (LinkageError | Exception e) { } catch (LinkageError | Exception e) {
// Fallback to next coder if possible // Fallback to next coder if possible
LOG.warn("Failed to create raw erasure decoder " + rawCoderName + LOG.warn("Failed to create raw erasure decoder " + factName +
", fallback to next codec if possible", e); ", fallback to next codec if possible", e);
} }
} }
throw new IllegalArgumentException("Fail to create raw erasure " + throw new IllegalArgumentException("Fail to create raw erasure " +
"encoder with given codec: " + codecName); "encoder with given codec: " + codec);
} }
private static ErasureCodec createCodec(Configuration conf, private static ErasureCodec createCodec(Configuration conf,

View File

@ -25,7 +25,6 @@ public final class ErasureCodeConstants {
private ErasureCodeConstants() { private ErasureCodeConstants() {
} }
public static final String DUMMY_CODEC_NAME = "dummy";
public static final String RS_CODEC_NAME = "rs"; public static final String RS_CODEC_NAME = "rs";
public static final String RS_LEGACY_CODEC_NAME = "rs-legacy"; public static final String RS_LEGACY_CODEC_NAME = "rs-legacy";
public static final String XOR_CODEC_NAME = "xor"; public static final String XOR_CODEC_NAME = "xor";

View File

@ -18,7 +18,6 @@
package org.apache.hadoop.io.erasurecode.rawcoder; package org.apache.hadoop.io.erasurecode.rawcoder;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
/** /**
@ -26,7 +25,6 @@ import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
*/ */
@InterfaceAudience.Private @InterfaceAudience.Private
public class DummyRawErasureCoderFactory implements RawErasureCoderFactory { public class DummyRawErasureCoderFactory implements RawErasureCoderFactory {
public static final String CODER_NAME = "dummy_dummy";
@Override @Override
public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) { public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) {
@ -37,14 +35,4 @@ public class DummyRawErasureCoderFactory implements RawErasureCoderFactory {
public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) { public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) {
return new DummyRawDecoder(coderOptions); return new DummyRawDecoder(coderOptions);
} }
@Override
public String getCoderName() {
return CODER_NAME;
}
@Override
public String getCodecName() {
return ErasureCodeConstants.DUMMY_CODEC_NAME;
}
} }

View File

@ -18,7 +18,6 @@
package org.apache.hadoop.io.erasurecode.rawcoder; package org.apache.hadoop.io.erasurecode.rawcoder;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
/** /**
@ -28,8 +27,6 @@ import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
@InterfaceAudience.Private @InterfaceAudience.Private
public class NativeRSRawErasureCoderFactory implements RawErasureCoderFactory { public class NativeRSRawErasureCoderFactory implements RawErasureCoderFactory {
public static final String CODER_NAME = "rs_native";
@Override @Override
public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) { public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) {
return new NativeRSRawEncoder(coderOptions); return new NativeRSRawEncoder(coderOptions);
@ -39,14 +36,4 @@ public class NativeRSRawErasureCoderFactory implements RawErasureCoderFactory {
public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) { public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) {
return new NativeRSRawDecoder(coderOptions); return new NativeRSRawDecoder(coderOptions);
} }
@Override
public String getCoderName() {
return CODER_NAME;
}
@Override
public String getCodecName() {
return ErasureCodeConstants.RS_CODEC_NAME;
}
} }

View File

@ -18,7 +18,6 @@
package org.apache.hadoop.io.erasurecode.rawcoder; package org.apache.hadoop.io.erasurecode.rawcoder;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
/** /**
@ -28,8 +27,6 @@ import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
@InterfaceAudience.Private @InterfaceAudience.Private
public class NativeXORRawErasureCoderFactory implements RawErasureCoderFactory { public class NativeXORRawErasureCoderFactory implements RawErasureCoderFactory {
public static final String CODER_NAME = "xor_native";
@Override @Override
public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) { public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) {
return new NativeXORRawEncoder(coderOptions); return new NativeXORRawEncoder(coderOptions);
@ -39,14 +36,4 @@ public class NativeXORRawErasureCoderFactory implements RawErasureCoderFactory {
public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) { public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) {
return new NativeXORRawDecoder(coderOptions); return new NativeXORRawDecoder(coderOptions);
} }
@Override
public String getCoderName() {
return CODER_NAME;
}
@Override
public String getCodecName() {
return ErasureCodeConstants.XOR_CODEC_NAME;
}
} }

View File

@ -18,7 +18,6 @@
package org.apache.hadoop.io.erasurecode.rawcoder; package org.apache.hadoop.io.erasurecode.rawcoder;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
/** /**
@ -27,8 +26,6 @@ import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
@InterfaceAudience.Private @InterfaceAudience.Private
public class RSLegacyRawErasureCoderFactory implements RawErasureCoderFactory { public class RSLegacyRawErasureCoderFactory implements RawErasureCoderFactory {
public static final String CODER_NAME = "rs-legacy_java";
@Override @Override
public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) { public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) {
return new RSLegacyRawEncoder(coderOptions); return new RSLegacyRawEncoder(coderOptions);
@ -38,14 +35,4 @@ public class RSLegacyRawErasureCoderFactory implements RawErasureCoderFactory {
public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) { public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) {
return new RSLegacyRawDecoder(coderOptions); return new RSLegacyRawDecoder(coderOptions);
} }
@Override
public String getCoderName() {
return CODER_NAME;
}
@Override
public String getCodecName() {
return ErasureCodeConstants.RS_LEGACY_CODEC_NAME;
}
} }

View File

@ -18,7 +18,6 @@
package org.apache.hadoop.io.erasurecode.rawcoder; package org.apache.hadoop.io.erasurecode.rawcoder;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
/** /**
@ -27,8 +26,6 @@ import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
@InterfaceAudience.Private @InterfaceAudience.Private
public class RSRawErasureCoderFactory implements RawErasureCoderFactory { public class RSRawErasureCoderFactory implements RawErasureCoderFactory {
public static final String CODER_NAME = "rs_java";
@Override @Override
public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) { public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) {
return new RSRawEncoder(coderOptions); return new RSRawEncoder(coderOptions);
@ -38,14 +35,4 @@ public class RSRawErasureCoderFactory implements RawErasureCoderFactory {
public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) { public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) {
return new RSRawDecoder(coderOptions); return new RSRawDecoder(coderOptions);
} }
@Override
public String getCoderName() {
return CODER_NAME;
}
@Override
public String getCodecName() {
return ErasureCodeConstants.RS_CODEC_NAME;
}
} }

View File

@ -41,16 +41,4 @@ public interface RawErasureCoderFactory {
* @return raw erasure decoder * @return raw erasure decoder
*/ */
RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions); RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions);
/**
* Get the name of the coder.
* @return coder name
*/
String getCoderName();
/**
* Get the name of its codec.
* @return codec name
*/
String getCodecName();
} }

View File

@ -18,7 +18,6 @@
package org.apache.hadoop.io.erasurecode.rawcoder; package org.apache.hadoop.io.erasurecode.rawcoder;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
/** /**
@ -27,8 +26,6 @@ import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
@InterfaceAudience.Private @InterfaceAudience.Private
public class XORRawErasureCoderFactory implements RawErasureCoderFactory { public class XORRawErasureCoderFactory implements RawErasureCoderFactory {
public static final String CODER_NAME = "xor_java";
@Override @Override
public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) { public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) {
return new XORRawEncoder(coderOptions); return new XORRawEncoder(coderOptions);
@ -38,14 +35,4 @@ public class XORRawErasureCoderFactory implements RawErasureCoderFactory {
public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) { public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) {
return new XORRawDecoder(coderOptions); return new XORRawDecoder(coderOptions);
} }
@Override
public String getCoderName() {
return CODER_NAME;
}
@Override
public String getCodecName() {
return ErasureCodeConstants.XOR_CODEC_NAME;
}
} }

View File

@ -668,7 +668,7 @@
<property> <property>
<name>io.erasurecode.codec.rs.rawcoders</name> <name>io.erasurecode.codec.rs.rawcoders</name>
<value>rs_native,rs_java</value> <value>org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory,org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory</value>
<description> <description>
Comma separated raw coder implementations for the rs codec. The earlier Comma separated raw coder implementations for the rs codec. The earlier
factory is prior to followings in case of failure of creating raw coders. factory is prior to followings in case of failure of creating raw coders.
@ -677,7 +677,7 @@
<property> <property>
<name>io.erasurecode.codec.rs-legacy.rawcoders</name> <name>io.erasurecode.codec.rs-legacy.rawcoders</name>
<value>rs-legacy_java</value> <value>org.apache.hadoop.io.erasurecode.rawcoder.RSLegacyRawErasureCoderFactory</value>
<description> <description>
Comma separated raw coder implementations for the rs-legacy codec. The earlier Comma separated raw coder implementations for the rs-legacy codec. The earlier
factory is prior to followings in case of failure of creating raw coders. factory is prior to followings in case of failure of creating raw coders.
@ -686,7 +686,7 @@
<property> <property>
<name>io.erasurecode.codec.xor.rawcoders</name> <name>io.erasurecode.codec.xor.rawcoders</name>
<value>xor_native,xor_java</value> <value>org.apache.hadoop.io.erasurecode.rawcoder.NativeXORRawErasureCoderFactory,org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory</value>
<description> <description>
Comma separated raw coder implementations for the xor codec. The earlier Comma separated raw coder implementations for the xor codec. The earlier
factory is prior to followings in case of failure of creating raw coders. factory is prior to followings in case of failure of creating raw coders.

View File

@ -28,7 +28,6 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
import org.apache.hadoop.io.erasurecode.rawcoder.XORRawDecoder; import org.apache.hadoop.io.erasurecode.rawcoder.XORRawDecoder;
import org.apache.hadoop.io.erasurecode.rawcoder.XORRawEncoder; import org.apache.hadoop.io.erasurecode.rawcoder.XORRawEncoder;
import org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory;
import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
@ -105,8 +104,8 @@ public class TestCodecRawCoderMapping {
ErasureCoderOptions coderOptions = new ErasureCoderOptions( ErasureCoderOptions coderOptions = new ErasureCoderOptions(
numDataUnit, numParityUnit); numDataUnit, numParityUnit);
conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY, conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
RSRawErasureCoderFactory.CODER_NAME + RSRawErasureCoderFactory.class.getCanonicalName() +
"," + NativeRSRawErasureCoderFactory.CODER_NAME); "," + NativeRSRawErasureCoderFactory.class.getCanonicalName());
// should return default raw coder of rs codec // should return default raw coder of rs codec
RawErasureEncoder encoder = CodecUtil.createRawEncoder( RawErasureEncoder encoder = CodecUtil.createRawEncoder(
conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions); conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions);
@ -134,7 +133,8 @@ public class TestCodecRawCoderMapping {
ErasureCoderOptions coderOptions = new ErasureCoderOptions( ErasureCoderOptions coderOptions = new ErasureCoderOptions(
numDataUnit, numParityUnit); numDataUnit, numParityUnit);
conf.set(CodecUtil.IO_ERASURECODE_CODEC_XOR_RAWCODERS_KEY, conf.set(CodecUtil.IO_ERASURECODE_CODEC_XOR_RAWCODERS_KEY,
"invalid-codec," + XORRawErasureCoderFactory.CODER_NAME); "invalid-codec," +
"org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory");
// should return second coder specified by IO_ERASURECODE_CODEC_CODERS // should return second coder specified by IO_ERASURECODE_CODEC_CODERS
RawErasureEncoder encoder = CodecUtil.createRawEncoder( RawErasureEncoder encoder = CodecUtil.createRawEncoder(
conf, ErasureCodeConstants.XOR_CODEC_NAME, coderOptions); conf, ErasureCodeConstants.XOR_CODEC_NAME, coderOptions);

View File

@ -51,7 +51,7 @@ public class TestHHXORErasureCoder extends TestHHErasureCoderBase {
*/ */
Configuration conf = new Configuration(); Configuration conf = new Configuration();
conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY, conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
RSRawErasureCoderFactory.CODER_NAME); RSRawErasureCoderFactory.class.getCanonicalName());
prepare(conf, 10, 4, new int[]{0}, new int[0]); prepare(conf, 10, 4, new int[]{0}, new int[0]);
testCoding(true); testCoding(true);

View File

@ -58,7 +58,7 @@ public class TestRSErasureCoder extends TestErasureCoderBase {
*/ */
Configuration conf = new Configuration(); Configuration conf = new Configuration();
conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY, conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
RSRawErasureCoderFactory.CODER_NAME); RSRawErasureCoderFactory.class.getCanonicalName());
prepare(conf, 10, 4, new int[]{0}, new int[0]); prepare(conf, 10, 4, new int[]{0}, new int[0]);
testCoding(true); testCoding(true);

View File

@ -117,15 +117,11 @@ Deployment
be more appropriate. If the administrator only cares about node-level fault-tolerance, `RS-10-4-64k` would still be appropriate as long as be more appropriate. If the administrator only cares about node-level fault-tolerance, `RS-10-4-64k` would still be appropriate as long as
there are at least 14 DataNodes in the cluster. there are at least 14 DataNodes in the cluster.
The codec implementations for Reed-Solomon and XOR can be configured with the following client and DataNode configuration keys: The codec implementation for Reed-Solomon and XOR can be configured with the following client and DataNode configuration keys:
`io.erasurecode.codec.rs.rawcoders` for the default RS codec, `io.erasurecode.codec.rs.rawcoder` for the default RS codec,
`io.erasurecode.codec.rs-legacy.rawcoders` for the legacy RS codec, `io.erasurecode.codec.rs-legacy.rawcoder` for the legacy RS codec,
`io.erasurecode.codec.xor.rawcoders` for the XOR codec. `io.erasurecode.codec.xor.rawcoder` for the XOR codec.
User can also configure self-defined codec with configuration key like: The default implementations for all of these codecs are pure Java. For default RS codec, there is also a native implementation which leverages Intel ISA-L library to improve the performance of codec. For XOR codec, a native implementation which leverages Intel ISA-L library to improve the performance of codec is also supported. Please refer to section "Enable Intel ISA-L" for more detail information.
`io.erasurecode.codec.self-defined-codec.rawcoders`.
The values for these key are lists of coder names with a fall-back mechanism.
All these codecs have implementations in pure Java. For default RS codec, there is also a native implementation which leverages Intel ISA-L library to improve the performance of codec. For XOR codec, a native implementation which leverages Intel ISA-L library to improve the performance of codec is also supported. Please refer to section "Enable Intel ISA-L" for more detail information.
The default implementation for RS Legacy is pure Java, and the default implementations for default RS and XOR are native implementations using Intel ISA-L library.
Erasure coding background recovery work on the DataNodes can also be tuned via the following configuration parameters: Erasure coding background recovery work on the DataNodes can also be tuned via the following configuration parameters:

View File

@ -98,7 +98,7 @@ public class TestDFSStripedInputStream {
if (ErasureCodeNative.isNativeCodeLoaded()) { if (ErasureCodeNative.isNativeCodeLoaded()) {
conf.set( conf.set(
CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY, CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
NativeRSRawErasureCoderFactory.CODER_NAME); NativeRSRawErasureCoderFactory.class.getCanonicalName());
} }
SimulatedFSDataset.setFactory(conf); SimulatedFSDataset.setFactory(conf);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes( cluster = new MiniDFSCluster.Builder(conf).numDataNodes(

View File

@ -85,7 +85,7 @@ public class TestDFSStripedOutputStream {
if (ErasureCodeNative.isNativeCodeLoaded()) { if (ErasureCodeNative.isNativeCodeLoaded()) {
conf.set( conf.set(
CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY, CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
NativeRSRawErasureCoderFactory.CODER_NAME); NativeRSRawErasureCoderFactory.class.getCanonicalName());
} }
DFSTestUtil.enableAllECPolicies(conf); DFSTestUtil.enableAllECPolicies(conf);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();

View File

@ -214,7 +214,7 @@ public class TestDFSStripedOutputStreamWithFailure {
if (ErasureCodeNative.isNativeCodeLoaded()) { if (ErasureCodeNative.isNativeCodeLoaded()) {
conf.set( conf.set(
CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY, CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
NativeRSRawErasureCoderFactory.CODER_NAME); NativeRSRawErasureCoderFactory.class.getCanonicalName());
} }
DFSTestUtil.enableAllECPolicies(conf); DFSTestUtil.enableAllECPolicies(conf);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();

View File

@ -100,7 +100,7 @@ public class TestReconstructStripedFile {
if (ErasureCodeNative.isNativeCodeLoaded()) { if (ErasureCodeNative.isNativeCodeLoaded()) {
conf.set( conf.set(
CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY, CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
NativeRSRawErasureCoderFactory.CODER_NAME); NativeRSRawErasureCoderFactory.class.getCanonicalName());
} }
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
StripedFileTestUtil.getDefaultECPolicy().getName()); StripedFileTestUtil.getDefaultECPolicy().getName());

View File

@ -68,7 +68,7 @@ public class TestUnsetAndChangeDirectoryEcPolicy {
if (ErasureCodeNative.isNativeCodeLoaded()) { if (ErasureCodeNative.isNativeCodeLoaded()) {
conf.set( conf.set(
CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY, CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
NativeRSRawErasureCoderFactory.CODER_NAME); NativeRSRawErasureCoderFactory.class.getCanonicalName());
} }
DFSTestUtil.enableAllECPolicies(conf); DFSTestUtil.enableAllECPolicies(conf);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes( cluster = new MiniDFSCluster.Builder(conf).numDataNodes(