Revert "HADOOP-13200. Implement customizable and configurable erasure coders. Contributed by Tim Yao."
This reverts commit 872088c6e7
.
This commit is contained in:
parent
872088c6e7
commit
ddaeb3e497
|
@ -18,6 +18,8 @@
|
||||||
package org.apache.hadoop.io.erasurecode;
|
package org.apache.hadoop.io.erasurecode;
|
||||||
|
|
||||||
import com.google.common.base.Preconditions;
|
import com.google.common.base.Preconditions;
|
||||||
|
import com.google.common.base.Splitter;
|
||||||
|
import com.google.common.collect.ImmutableMap;
|
||||||
import org.apache.commons.logging.Log;
|
import org.apache.commons.logging.Log;
|
||||||
import org.apache.commons.logging.LogFactory;
|
import org.apache.commons.logging.LogFactory;
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
|
@ -28,12 +30,18 @@ import org.apache.hadoop.io.erasurecode.codec.RSErasureCodec;
|
||||||
import org.apache.hadoop.io.erasurecode.codec.XORErasureCodec;
|
import org.apache.hadoop.io.erasurecode.codec.XORErasureCodec;
|
||||||
import org.apache.hadoop.io.erasurecode.coder.ErasureDecoder;
|
import org.apache.hadoop.io.erasurecode.coder.ErasureDecoder;
|
||||||
import org.apache.hadoop.io.erasurecode.coder.ErasureEncoder;
|
import org.apache.hadoop.io.erasurecode.coder.ErasureEncoder;
|
||||||
|
import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory;
|
||||||
|
import org.apache.hadoop.io.erasurecode.rawcoder.NativeXORRawErasureCoderFactory;
|
||||||
|
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory;
|
||||||
|
import org.apache.hadoop.io.erasurecode.rawcoder.RSLegacyRawErasureCoderFactory;
|
||||||
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureCoderFactory;
|
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureCoderFactory;
|
||||||
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
|
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
|
||||||
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
|
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
|
||||||
|
import org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory;
|
||||||
|
|
||||||
import java.lang.reflect.Constructor;
|
import java.lang.reflect.Constructor;
|
||||||
import java.lang.reflect.InvocationTargetException;
|
import java.lang.reflect.InvocationTargetException;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A codec & coder utility to help create coders conveniently.
|
* A codec & coder utility to help create coders conveniently.
|
||||||
|
@ -71,12 +79,27 @@ public final class CodecUtil {
|
||||||
/** Comma separated raw codec name. The first coder is prior to the latter. */
|
/** Comma separated raw codec name. The first coder is prior to the latter. */
|
||||||
public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODERS_KEY =
|
public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODERS_KEY =
|
||||||
IO_ERASURECODE_CODEC + "rs-legacy.rawcoders";
|
IO_ERASURECODE_CODEC + "rs-legacy.rawcoders";
|
||||||
|
public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODERS_DEFAULT =
|
||||||
|
RSLegacyRawErasureCoderFactory.class.getCanonicalName();
|
||||||
public static final String IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY =
|
public static final String IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY =
|
||||||
IO_ERASURECODE_CODEC + "rs.rawcoders";
|
IO_ERASURECODE_CODEC + "rs.rawcoders";
|
||||||
|
public static final String IO_ERASURECODE_CODEC_RS_RAWCODERS_DEFAULT =
|
||||||
|
NativeRSRawErasureCoderFactory.class.getCanonicalName() +
|
||||||
|
"," + RSRawErasureCoderFactory.class.getCanonicalName();
|
||||||
|
|
||||||
/** Raw coder factory for the XOR codec. */
|
/** Raw coder factory for the XOR codec. */
|
||||||
public static final String IO_ERASURECODE_CODEC_XOR_RAWCODERS_KEY =
|
public static final String IO_ERASURECODE_CODEC_XOR_RAWCODERS_KEY =
|
||||||
IO_ERASURECODE_CODEC + "xor.rawcoders";
|
IO_ERASURECODE_CODEC + "xor.rawcoders";
|
||||||
|
public static final String IO_ERASURECODE_CODEC_XOR_RAWCODERS_DEFAULT =
|
||||||
|
NativeXORRawErasureCoderFactory.class.getCanonicalName() +
|
||||||
|
"," + XORRawErasureCoderFactory.class.getCanonicalName();
|
||||||
|
|
||||||
|
// Default coders for each codec names.
|
||||||
|
public static final Map<String, String> DEFAULT_CODERS_MAP = ImmutableMap.of(
|
||||||
|
"rs", IO_ERASURECODE_CODEC_RS_RAWCODERS_DEFAULT,
|
||||||
|
"rs-legacy", IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODERS_DEFAULT,
|
||||||
|
"xor", IO_ERASURECODE_CODEC_XOR_RAWCODERS_DEFAULT
|
||||||
|
);
|
||||||
|
|
||||||
private CodecUtil() { }
|
private CodecUtil() { }
|
||||||
|
|
||||||
|
@ -145,61 +168,70 @@ public final class CodecUtil {
|
||||||
}
|
}
|
||||||
|
|
||||||
private static RawErasureCoderFactory createRawCoderFactory(
|
private static RawErasureCoderFactory createRawCoderFactory(
|
||||||
String coderName, String codecName) {
|
Configuration conf, String rawCoderFactoryKey) {
|
||||||
RawErasureCoderFactory fact;
|
RawErasureCoderFactory fact;
|
||||||
fact = CodecRegistry.getInstance().
|
try {
|
||||||
getCoderByName(codecName, coderName);
|
Class<? extends RawErasureCoderFactory> factClass = conf.getClassByName(
|
||||||
|
rawCoderFactoryKey).asSubclass(RawErasureCoderFactory.class);
|
||||||
|
fact = factClass.newInstance();
|
||||||
|
} catch (ClassNotFoundException | InstantiationException |
|
||||||
|
IllegalAccessException e) {
|
||||||
|
throw new RuntimeException("Failed to create raw coder factory", e);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (fact == null) {
|
||||||
|
throw new RuntimeException("Failed to create raw coder factory");
|
||||||
|
}
|
||||||
|
|
||||||
return fact;
|
return fact;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Return a list of coder names
|
// Return comma separated coder names
|
||||||
private static String[] getRawCoderNames(
|
private static String getRawCoders(Configuration conf, String codec) {
|
||||||
Configuration conf, String codecName) {
|
return conf.get(
|
||||||
return conf.getStrings(
|
IO_ERASURECODE_CODEC + codec + ".rawcoders",
|
||||||
IO_ERASURECODE_CODEC + codecName + ".rawcoders",
|
DEFAULT_CODERS_MAP.getOrDefault(codec, codec)
|
||||||
CodecRegistry.getInstance().getCoderNames(codecName)
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static RawErasureEncoder createRawEncoderWithFallback(
|
private static RawErasureEncoder createRawEncoderWithFallback(
|
||||||
Configuration conf, String codecName, ErasureCoderOptions coderOptions) {
|
Configuration conf, String codec, ErasureCoderOptions coderOptions) {
|
||||||
String[] rawCoderNames = getRawCoderNames(conf, codecName);
|
String coders = getRawCoders(conf, codec);
|
||||||
for (String rawCoderName : rawCoderNames) {
|
for (String factName : Splitter.on(",").split(coders)) {
|
||||||
try {
|
try {
|
||||||
if (rawCoderName != null) {
|
if (factName != null) {
|
||||||
RawErasureCoderFactory fact = createRawCoderFactory(
|
RawErasureCoderFactory fact = createRawCoderFactory(conf,
|
||||||
rawCoderName, codecName);
|
factName);
|
||||||
return fact.createEncoder(coderOptions);
|
return fact.createEncoder(coderOptions);
|
||||||
}
|
}
|
||||||
} catch (LinkageError | Exception e) {
|
} catch (LinkageError | Exception e) {
|
||||||
// Fallback to next coder if possible
|
// Fallback to next coder if possible
|
||||||
LOG.warn("Failed to create raw erasure encoder " + rawCoderName +
|
LOG.warn("Failed to create raw erasure encoder " + factName +
|
||||||
", fallback to next codec if possible", e);
|
", fallback to next codec if possible", e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
throw new IllegalArgumentException("Fail to create raw erasure " +
|
throw new IllegalArgumentException("Fail to create raw erasure " +
|
||||||
"encoder with given codec: " + codecName);
|
"encoder with given codec: " + codec);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static RawErasureDecoder createRawDecoderWithFallback(
|
private static RawErasureDecoder createRawDecoderWithFallback(
|
||||||
Configuration conf, String codecName, ErasureCoderOptions coderOptions) {
|
Configuration conf, String codec, ErasureCoderOptions coderOptions) {
|
||||||
String[] coders = getRawCoderNames(conf, codecName);
|
String coders = getRawCoders(conf, codec);
|
||||||
for (String rawCoderName : coders) {
|
for (String factName : Splitter.on(",").split(coders)) {
|
||||||
try {
|
try {
|
||||||
if (rawCoderName != null) {
|
if (factName != null) {
|
||||||
RawErasureCoderFactory fact = createRawCoderFactory(
|
RawErasureCoderFactory fact = createRawCoderFactory(conf,
|
||||||
rawCoderName, codecName);
|
factName);
|
||||||
return fact.createDecoder(coderOptions);
|
return fact.createDecoder(coderOptions);
|
||||||
}
|
}
|
||||||
} catch (LinkageError | Exception e) {
|
} catch (LinkageError | Exception e) {
|
||||||
// Fallback to next coder if possible
|
// Fallback to next coder if possible
|
||||||
LOG.warn("Failed to create raw erasure decoder " + rawCoderName +
|
LOG.warn("Failed to create raw erasure decoder " + factName +
|
||||||
", fallback to next codec if possible", e);
|
", fallback to next codec if possible", e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
throw new IllegalArgumentException("Fail to create raw erasure " +
|
throw new IllegalArgumentException("Fail to create raw erasure " +
|
||||||
"encoder with given codec: " + codecName);
|
"encoder with given codec: " + codec);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static ErasureCodec createCodec(Configuration conf,
|
private static ErasureCodec createCodec(Configuration conf,
|
||||||
|
|
|
@ -25,7 +25,6 @@ public final class ErasureCodeConstants {
|
||||||
private ErasureCodeConstants() {
|
private ErasureCodeConstants() {
|
||||||
}
|
}
|
||||||
|
|
||||||
public static final String DUMMY_CODEC_NAME = "dummy";
|
|
||||||
public static final String RS_CODEC_NAME = "rs";
|
public static final String RS_CODEC_NAME = "rs";
|
||||||
public static final String RS_LEGACY_CODEC_NAME = "rs-legacy";
|
public static final String RS_LEGACY_CODEC_NAME = "rs-legacy";
|
||||||
public static final String XOR_CODEC_NAME = "xor";
|
public static final String XOR_CODEC_NAME = "xor";
|
||||||
|
|
|
@ -18,7 +18,6 @@
|
||||||
package org.apache.hadoop.io.erasurecode.rawcoder;
|
package org.apache.hadoop.io.erasurecode.rawcoder;
|
||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
|
|
||||||
import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
|
import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -26,7 +25,6 @@ import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
|
||||||
*/
|
*/
|
||||||
@InterfaceAudience.Private
|
@InterfaceAudience.Private
|
||||||
public class DummyRawErasureCoderFactory implements RawErasureCoderFactory {
|
public class DummyRawErasureCoderFactory implements RawErasureCoderFactory {
|
||||||
public static final String CODER_NAME = "dummy_dummy";
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) {
|
public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) {
|
||||||
|
@ -37,14 +35,4 @@ public class DummyRawErasureCoderFactory implements RawErasureCoderFactory {
|
||||||
public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) {
|
public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) {
|
||||||
return new DummyRawDecoder(coderOptions);
|
return new DummyRawDecoder(coderOptions);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
|
||||||
public String getCoderName() {
|
|
||||||
return CODER_NAME;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public String getCodecName() {
|
|
||||||
return ErasureCodeConstants.DUMMY_CODEC_NAME;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -18,7 +18,6 @@
|
||||||
package org.apache.hadoop.io.erasurecode.rawcoder;
|
package org.apache.hadoop.io.erasurecode.rawcoder;
|
||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
|
|
||||||
import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
|
import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -28,8 +27,6 @@ import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
|
||||||
@InterfaceAudience.Private
|
@InterfaceAudience.Private
|
||||||
public class NativeRSRawErasureCoderFactory implements RawErasureCoderFactory {
|
public class NativeRSRawErasureCoderFactory implements RawErasureCoderFactory {
|
||||||
|
|
||||||
public static final String CODER_NAME = "rs_native";
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) {
|
public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) {
|
||||||
return new NativeRSRawEncoder(coderOptions);
|
return new NativeRSRawEncoder(coderOptions);
|
||||||
|
@ -39,14 +36,4 @@ public class NativeRSRawErasureCoderFactory implements RawErasureCoderFactory {
|
||||||
public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) {
|
public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) {
|
||||||
return new NativeRSRawDecoder(coderOptions);
|
return new NativeRSRawDecoder(coderOptions);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
|
||||||
public String getCoderName() {
|
|
||||||
return CODER_NAME;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public String getCodecName() {
|
|
||||||
return ErasureCodeConstants.RS_CODEC_NAME;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -18,7 +18,6 @@
|
||||||
package org.apache.hadoop.io.erasurecode.rawcoder;
|
package org.apache.hadoop.io.erasurecode.rawcoder;
|
||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
|
|
||||||
import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
|
import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -28,8 +27,6 @@ import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
|
||||||
@InterfaceAudience.Private
|
@InterfaceAudience.Private
|
||||||
public class NativeXORRawErasureCoderFactory implements RawErasureCoderFactory {
|
public class NativeXORRawErasureCoderFactory implements RawErasureCoderFactory {
|
||||||
|
|
||||||
public static final String CODER_NAME = "xor_native";
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) {
|
public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) {
|
||||||
return new NativeXORRawEncoder(coderOptions);
|
return new NativeXORRawEncoder(coderOptions);
|
||||||
|
@ -39,14 +36,4 @@ public class NativeXORRawErasureCoderFactory implements RawErasureCoderFactory {
|
||||||
public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) {
|
public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) {
|
||||||
return new NativeXORRawDecoder(coderOptions);
|
return new NativeXORRawDecoder(coderOptions);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
|
||||||
public String getCoderName() {
|
|
||||||
return CODER_NAME;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public String getCodecName() {
|
|
||||||
return ErasureCodeConstants.XOR_CODEC_NAME;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -18,7 +18,6 @@
|
||||||
package org.apache.hadoop.io.erasurecode.rawcoder;
|
package org.apache.hadoop.io.erasurecode.rawcoder;
|
||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
|
|
||||||
import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
|
import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -27,8 +26,6 @@ import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
|
||||||
@InterfaceAudience.Private
|
@InterfaceAudience.Private
|
||||||
public class RSLegacyRawErasureCoderFactory implements RawErasureCoderFactory {
|
public class RSLegacyRawErasureCoderFactory implements RawErasureCoderFactory {
|
||||||
|
|
||||||
public static final String CODER_NAME = "rs-legacy_java";
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) {
|
public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) {
|
||||||
return new RSLegacyRawEncoder(coderOptions);
|
return new RSLegacyRawEncoder(coderOptions);
|
||||||
|
@ -38,14 +35,4 @@ public class RSLegacyRawErasureCoderFactory implements RawErasureCoderFactory {
|
||||||
public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) {
|
public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) {
|
||||||
return new RSLegacyRawDecoder(coderOptions);
|
return new RSLegacyRawDecoder(coderOptions);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
|
||||||
public String getCoderName() {
|
|
||||||
return CODER_NAME;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public String getCodecName() {
|
|
||||||
return ErasureCodeConstants.RS_LEGACY_CODEC_NAME;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -18,7 +18,6 @@
|
||||||
package org.apache.hadoop.io.erasurecode.rawcoder;
|
package org.apache.hadoop.io.erasurecode.rawcoder;
|
||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
|
|
||||||
import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
|
import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -27,8 +26,6 @@ import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
|
||||||
@InterfaceAudience.Private
|
@InterfaceAudience.Private
|
||||||
public class RSRawErasureCoderFactory implements RawErasureCoderFactory {
|
public class RSRawErasureCoderFactory implements RawErasureCoderFactory {
|
||||||
|
|
||||||
public static final String CODER_NAME = "rs_java";
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) {
|
public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) {
|
||||||
return new RSRawEncoder(coderOptions);
|
return new RSRawEncoder(coderOptions);
|
||||||
|
@ -38,14 +35,4 @@ public class RSRawErasureCoderFactory implements RawErasureCoderFactory {
|
||||||
public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) {
|
public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) {
|
||||||
return new RSRawDecoder(coderOptions);
|
return new RSRawDecoder(coderOptions);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
|
||||||
public String getCoderName() {
|
|
||||||
return CODER_NAME;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public String getCodecName() {
|
|
||||||
return ErasureCodeConstants.RS_CODEC_NAME;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -41,16 +41,4 @@ public interface RawErasureCoderFactory {
|
||||||
* @return raw erasure decoder
|
* @return raw erasure decoder
|
||||||
*/
|
*/
|
||||||
RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions);
|
RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions);
|
||||||
|
|
||||||
/**
|
|
||||||
* Get the name of the coder.
|
|
||||||
* @return coder name
|
|
||||||
*/
|
|
||||||
String getCoderName();
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get the name of its codec.
|
|
||||||
* @return codec name
|
|
||||||
*/
|
|
||||||
String getCodecName();
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -18,7 +18,6 @@
|
||||||
package org.apache.hadoop.io.erasurecode.rawcoder;
|
package org.apache.hadoop.io.erasurecode.rawcoder;
|
||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
|
|
||||||
import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
|
import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -27,8 +26,6 @@ import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
|
||||||
@InterfaceAudience.Private
|
@InterfaceAudience.Private
|
||||||
public class XORRawErasureCoderFactory implements RawErasureCoderFactory {
|
public class XORRawErasureCoderFactory implements RawErasureCoderFactory {
|
||||||
|
|
||||||
public static final String CODER_NAME = "xor_java";
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) {
|
public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) {
|
||||||
return new XORRawEncoder(coderOptions);
|
return new XORRawEncoder(coderOptions);
|
||||||
|
@ -38,14 +35,4 @@ public class XORRawErasureCoderFactory implements RawErasureCoderFactory {
|
||||||
public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) {
|
public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) {
|
||||||
return new XORRawDecoder(coderOptions);
|
return new XORRawDecoder(coderOptions);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
|
||||||
public String getCoderName() {
|
|
||||||
return CODER_NAME;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public String getCodecName() {
|
|
||||||
return ErasureCodeConstants.XOR_CODEC_NAME;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -668,7 +668,7 @@
|
||||||
|
|
||||||
<property>
|
<property>
|
||||||
<name>io.erasurecode.codec.rs.rawcoders</name>
|
<name>io.erasurecode.codec.rs.rawcoders</name>
|
||||||
<value>rs_native,rs_java</value>
|
<value>org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory,org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory</value>
|
||||||
<description>
|
<description>
|
||||||
Comma separated raw coder implementations for the rs codec. The earlier
|
Comma separated raw coder implementations for the rs codec. The earlier
|
||||||
factory is prior to followings in case of failure of creating raw coders.
|
factory is prior to followings in case of failure of creating raw coders.
|
||||||
|
@ -677,7 +677,7 @@
|
||||||
|
|
||||||
<property>
|
<property>
|
||||||
<name>io.erasurecode.codec.rs-legacy.rawcoders</name>
|
<name>io.erasurecode.codec.rs-legacy.rawcoders</name>
|
||||||
<value>rs-legacy_java</value>
|
<value>org.apache.hadoop.io.erasurecode.rawcoder.RSLegacyRawErasureCoderFactory</value>
|
||||||
<description>
|
<description>
|
||||||
Comma separated raw coder implementations for the rs-legacy codec. The earlier
|
Comma separated raw coder implementations for the rs-legacy codec. The earlier
|
||||||
factory is prior to followings in case of failure of creating raw coders.
|
factory is prior to followings in case of failure of creating raw coders.
|
||||||
|
@ -686,7 +686,7 @@
|
||||||
|
|
||||||
<property>
|
<property>
|
||||||
<name>io.erasurecode.codec.xor.rawcoders</name>
|
<name>io.erasurecode.codec.xor.rawcoders</name>
|
||||||
<value>xor_native,xor_java</value>
|
<value>org.apache.hadoop.io.erasurecode.rawcoder.NativeXORRawErasureCoderFactory,org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory</value>
|
||||||
<description>
|
<description>
|
||||||
Comma separated raw coder implementations for the xor codec. The earlier
|
Comma separated raw coder implementations for the xor codec. The earlier
|
||||||
factory is prior to followings in case of failure of creating raw coders.
|
factory is prior to followings in case of failure of creating raw coders.
|
||||||
|
|
|
@ -28,7 +28,6 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
|
||||||
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
|
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
|
||||||
import org.apache.hadoop.io.erasurecode.rawcoder.XORRawDecoder;
|
import org.apache.hadoop.io.erasurecode.rawcoder.XORRawDecoder;
|
||||||
import org.apache.hadoop.io.erasurecode.rawcoder.XORRawEncoder;
|
import org.apache.hadoop.io.erasurecode.rawcoder.XORRawEncoder;
|
||||||
import org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory;
|
|
||||||
import org.apache.hadoop.test.GenericTestUtils;
|
import org.apache.hadoop.test.GenericTestUtils;
|
||||||
import org.junit.Assert;
|
import org.junit.Assert;
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
|
@ -105,8 +104,8 @@ public class TestCodecRawCoderMapping {
|
||||||
ErasureCoderOptions coderOptions = new ErasureCoderOptions(
|
ErasureCoderOptions coderOptions = new ErasureCoderOptions(
|
||||||
numDataUnit, numParityUnit);
|
numDataUnit, numParityUnit);
|
||||||
conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
|
conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
|
||||||
RSRawErasureCoderFactory.CODER_NAME +
|
RSRawErasureCoderFactory.class.getCanonicalName() +
|
||||||
"," + NativeRSRawErasureCoderFactory.CODER_NAME);
|
"," + NativeRSRawErasureCoderFactory.class.getCanonicalName());
|
||||||
// should return default raw coder of rs codec
|
// should return default raw coder of rs codec
|
||||||
RawErasureEncoder encoder = CodecUtil.createRawEncoder(
|
RawErasureEncoder encoder = CodecUtil.createRawEncoder(
|
||||||
conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions);
|
conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions);
|
||||||
|
@ -134,7 +133,8 @@ public class TestCodecRawCoderMapping {
|
||||||
ErasureCoderOptions coderOptions = new ErasureCoderOptions(
|
ErasureCoderOptions coderOptions = new ErasureCoderOptions(
|
||||||
numDataUnit, numParityUnit);
|
numDataUnit, numParityUnit);
|
||||||
conf.set(CodecUtil.IO_ERASURECODE_CODEC_XOR_RAWCODERS_KEY,
|
conf.set(CodecUtil.IO_ERASURECODE_CODEC_XOR_RAWCODERS_KEY,
|
||||||
"invalid-codec," + XORRawErasureCoderFactory.CODER_NAME);
|
"invalid-codec," +
|
||||||
|
"org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory");
|
||||||
// should return second coder specified by IO_ERASURECODE_CODEC_CODERS
|
// should return second coder specified by IO_ERASURECODE_CODEC_CODERS
|
||||||
RawErasureEncoder encoder = CodecUtil.createRawEncoder(
|
RawErasureEncoder encoder = CodecUtil.createRawEncoder(
|
||||||
conf, ErasureCodeConstants.XOR_CODEC_NAME, coderOptions);
|
conf, ErasureCodeConstants.XOR_CODEC_NAME, coderOptions);
|
||||||
|
|
|
@ -51,7 +51,7 @@ public class TestHHXORErasureCoder extends TestHHErasureCoderBase {
|
||||||
*/
|
*/
|
||||||
Configuration conf = new Configuration();
|
Configuration conf = new Configuration();
|
||||||
conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
|
conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
|
||||||
RSRawErasureCoderFactory.CODER_NAME);
|
RSRawErasureCoderFactory.class.getCanonicalName());
|
||||||
prepare(conf, 10, 4, new int[]{0}, new int[0]);
|
prepare(conf, 10, 4, new int[]{0}, new int[0]);
|
||||||
|
|
||||||
testCoding(true);
|
testCoding(true);
|
||||||
|
|
|
@ -58,7 +58,7 @@ public class TestRSErasureCoder extends TestErasureCoderBase {
|
||||||
*/
|
*/
|
||||||
Configuration conf = new Configuration();
|
Configuration conf = new Configuration();
|
||||||
conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
|
conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
|
||||||
RSRawErasureCoderFactory.CODER_NAME);
|
RSRawErasureCoderFactory.class.getCanonicalName());
|
||||||
prepare(conf, 10, 4, new int[]{0}, new int[0]);
|
prepare(conf, 10, 4, new int[]{0}, new int[0]);
|
||||||
|
|
||||||
testCoding(true);
|
testCoding(true);
|
||||||
|
|
|
@ -117,15 +117,11 @@ Deployment
|
||||||
be more appropriate. If the administrator only cares about node-level fault-tolerance, `RS-10-4-64k` would still be appropriate as long as
|
be more appropriate. If the administrator only cares about node-level fault-tolerance, `RS-10-4-64k` would still be appropriate as long as
|
||||||
there are at least 14 DataNodes in the cluster.
|
there are at least 14 DataNodes in the cluster.
|
||||||
|
|
||||||
The codec implementations for Reed-Solomon and XOR can be configured with the following client and DataNode configuration keys:
|
The codec implementation for Reed-Solomon and XOR can be configured with the following client and DataNode configuration keys:
|
||||||
`io.erasurecode.codec.rs.rawcoders` for the default RS codec,
|
`io.erasurecode.codec.rs.rawcoder` for the default RS codec,
|
||||||
`io.erasurecode.codec.rs-legacy.rawcoders` for the legacy RS codec,
|
`io.erasurecode.codec.rs-legacy.rawcoder` for the legacy RS codec,
|
||||||
`io.erasurecode.codec.xor.rawcoders` for the XOR codec.
|
`io.erasurecode.codec.xor.rawcoder` for the XOR codec.
|
||||||
User can also configure self-defined codec with configuration key like:
|
The default implementations for all of these codecs are pure Java. For default RS codec, there is also a native implementation which leverages Intel ISA-L library to improve the performance of codec. For XOR codec, a native implementation which leverages Intel ISA-L library to improve the performance of codec is also supported. Please refer to section "Enable Intel ISA-L" for more detail information.
|
||||||
`io.erasurecode.codec.self-defined-codec.rawcoders`.
|
|
||||||
The values for these key are lists of coder names with a fall-back mechanism.
|
|
||||||
All these codecs have implementations in pure Java. For default RS codec, there is also a native implementation which leverages Intel ISA-L library to improve the performance of codec. For XOR codec, a native implementation which leverages Intel ISA-L library to improve the performance of codec is also supported. Please refer to section "Enable Intel ISA-L" for more detail information.
|
|
||||||
The default implementation for RS Legacy is pure Java, and the default implementations for default RS and XOR are native implementations using Intel ISA-L library.
|
|
||||||
|
|
||||||
Erasure coding background recovery work on the DataNodes can also be tuned via the following configuration parameters:
|
Erasure coding background recovery work on the DataNodes can also be tuned via the following configuration parameters:
|
||||||
|
|
||||||
|
|
|
@ -98,7 +98,7 @@ public class TestDFSStripedInputStream {
|
||||||
if (ErasureCodeNative.isNativeCodeLoaded()) {
|
if (ErasureCodeNative.isNativeCodeLoaded()) {
|
||||||
conf.set(
|
conf.set(
|
||||||
CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
|
CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
|
||||||
NativeRSRawErasureCoderFactory.CODER_NAME);
|
NativeRSRawErasureCoderFactory.class.getCanonicalName());
|
||||||
}
|
}
|
||||||
SimulatedFSDataset.setFactory(conf);
|
SimulatedFSDataset.setFactory(conf);
|
||||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(
|
||||||
|
|
|
@ -85,7 +85,7 @@ public class TestDFSStripedOutputStream {
|
||||||
if (ErasureCodeNative.isNativeCodeLoaded()) {
|
if (ErasureCodeNative.isNativeCodeLoaded()) {
|
||||||
conf.set(
|
conf.set(
|
||||||
CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
|
CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
|
||||||
NativeRSRawErasureCoderFactory.CODER_NAME);
|
NativeRSRawErasureCoderFactory.class.getCanonicalName());
|
||||||
}
|
}
|
||||||
DFSTestUtil.enableAllECPolicies(conf);
|
DFSTestUtil.enableAllECPolicies(conf);
|
||||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
|
||||||
|
|
|
@ -214,7 +214,7 @@ public class TestDFSStripedOutputStreamWithFailure {
|
||||||
if (ErasureCodeNative.isNativeCodeLoaded()) {
|
if (ErasureCodeNative.isNativeCodeLoaded()) {
|
||||||
conf.set(
|
conf.set(
|
||||||
CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
|
CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
|
||||||
NativeRSRawErasureCoderFactory.CODER_NAME);
|
NativeRSRawErasureCoderFactory.class.getCanonicalName());
|
||||||
}
|
}
|
||||||
DFSTestUtil.enableAllECPolicies(conf);
|
DFSTestUtil.enableAllECPolicies(conf);
|
||||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
|
||||||
|
|
|
@ -100,7 +100,7 @@ public class TestReconstructStripedFile {
|
||||||
if (ErasureCodeNative.isNativeCodeLoaded()) {
|
if (ErasureCodeNative.isNativeCodeLoaded()) {
|
||||||
conf.set(
|
conf.set(
|
||||||
CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
|
CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
|
||||||
NativeRSRawErasureCoderFactory.CODER_NAME);
|
NativeRSRawErasureCoderFactory.class.getCanonicalName());
|
||||||
}
|
}
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
|
|
|
@ -68,7 +68,7 @@ public class TestUnsetAndChangeDirectoryEcPolicy {
|
||||||
if (ErasureCodeNative.isNativeCodeLoaded()) {
|
if (ErasureCodeNative.isNativeCodeLoaded()) {
|
||||||
conf.set(
|
conf.set(
|
||||||
CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
|
CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
|
||||||
NativeRSRawErasureCoderFactory.CODER_NAME);
|
NativeRSRawErasureCoderFactory.class.getCanonicalName());
|
||||||
}
|
}
|
||||||
DFSTestUtil.enableAllECPolicies(conf);
|
DFSTestUtil.enableAllECPolicies(conf);
|
||||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(
|
||||||
|
|
Loading…
Reference in New Issue