HADOOP-14261. Some refactoring work for erasure coding raw coder. Contributed by Lin Zeng.
This commit is contained in:
parent
b0803388fc
commit
a22fe02fba
|
@ -33,7 +33,7 @@ import org.apache.hadoop.io.erasurecode.coder.ErasureEncoder;
|
|||
import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory;
|
||||
import org.apache.hadoop.io.erasurecode.rawcoder.NativeXORRawErasureCoderFactory;
|
||||
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory;
|
||||
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactoryLegacy;
|
||||
import org.apache.hadoop.io.erasurecode.rawcoder.RSLegacyRawErasureCoderFactory;
|
||||
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureCoderFactory;
|
||||
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
|
||||
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
|
||||
|
@ -80,7 +80,7 @@ public final class CodecUtil {
|
|||
public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODERS_KEY =
|
||||
IO_ERASURECODE_CODEC + "rs-legacy.rawcoders";
|
||||
public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODERS_DEFAULT =
|
||||
RSRawErasureCoderFactoryLegacy.class.getCanonicalName();
|
||||
RSLegacyRawErasureCoderFactory.class.getCanonicalName();
|
||||
public static final String IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY =
|
||||
IO_ERASURECODE_CODEC + "rs.rawcoders";
|
||||
public static final String IO_ERASURECODE_CODEC_RS_RAWCODERS_DEFAULT =
|
||||
|
|
|
@ -34,12 +34,12 @@ import java.nio.ByteBuffer;
|
|||
* addressed in HADOOP-11871.
|
||||
*/
|
||||
@InterfaceAudience.Private
|
||||
public class RSRawDecoderLegacy extends RawErasureDecoder {
|
||||
public class RSLegacyRawDecoder extends RawErasureDecoder {
|
||||
// To describe and calculate the needed Vandermonde matrix
|
||||
private int[] errSignature;
|
||||
private int[] primitivePower;
|
||||
|
||||
public RSRawDecoderLegacy(ErasureCoderOptions coderOptions) {
|
||||
public RSLegacyRawDecoder(ErasureCoderOptions coderOptions) {
|
||||
super(coderOptions);
|
||||
if (getNumAllUnits() >= RSUtil.GF.getFieldSize()) {
|
||||
throw new HadoopIllegalArgumentException(
|
|
@ -30,10 +30,10 @@ import java.util.Arrays;
|
|||
* when possible.
|
||||
*/
|
||||
@InterfaceAudience.Private
|
||||
public class RSRawEncoderLegacy extends RawErasureEncoder {
|
||||
public class RSLegacyRawEncoder extends RawErasureEncoder {
|
||||
private int[] generatingPolynomial;
|
||||
|
||||
public RSRawEncoderLegacy(ErasureCoderOptions coderOptions) {
|
||||
public RSLegacyRawEncoder(ErasureCoderOptions coderOptions) {
|
||||
super(coderOptions);
|
||||
|
||||
assert (getNumDataUnits() + getNumParityUnits() < RSUtil.GF.getFieldSize());
|
|
@ -24,15 +24,15 @@ import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
|
|||
* A raw coder factory for the legacy raw Reed-Solomon coder in Java.
|
||||
*/
|
||||
@InterfaceAudience.Private
|
||||
public class RSRawErasureCoderFactoryLegacy implements RawErasureCoderFactory {
|
||||
public class RSLegacyRawErasureCoderFactory implements RawErasureCoderFactory {
|
||||
|
||||
@Override
|
||||
public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) {
|
||||
return new RSRawEncoderLegacy(coderOptions);
|
||||
return new RSLegacyRawEncoder(coderOptions);
|
||||
}
|
||||
|
||||
@Override
|
||||
public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) {
|
||||
return new RSRawDecoderLegacy(coderOptions);
|
||||
return new RSLegacyRawDecoder(coderOptions);
|
||||
}
|
||||
}
|
|
@ -677,7 +677,7 @@
|
|||
|
||||
<property>
|
||||
<name>io.erasurecode.codec.rs-legacy.rawcoders</name>
|
||||
<value>org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactoryLegacy</value>
|
||||
<value>org.apache.hadoop.io.erasurecode.rawcoder.RSLegacyRawErasureCoderFactory</value>
|
||||
<description>
|
||||
Comma separated raw coder implementations for the rs-legacy codec. The earlier
|
||||
factory is prior to followings in case of failure of creating raw coders.
|
||||
|
|
|
@ -20,9 +20,9 @@ package org.apache.hadoop.io.erasurecode;
|
|||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory;
|
||||
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawDecoder;
|
||||
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawDecoderLegacy;
|
||||
import org.apache.hadoop.io.erasurecode.rawcoder.RSLegacyRawDecoder;
|
||||
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawEncoder;
|
||||
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawEncoderLegacy;
|
||||
import org.apache.hadoop.io.erasurecode.rawcoder.RSLegacyRawEncoder;
|
||||
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory;
|
||||
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
|
||||
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
|
||||
|
@ -62,10 +62,10 @@ public class TestCodecRawCoderMapping {
|
|||
// should return default raw coder of rs-legacy codec
|
||||
encoder = CodecUtil.createRawEncoder(conf,
|
||||
ErasureCodeConstants.RS_LEGACY_CODEC_NAME, coderOptions);
|
||||
Assert.assertTrue(encoder instanceof RSRawEncoderLegacy);
|
||||
Assert.assertTrue(encoder instanceof RSLegacyRawEncoder);
|
||||
decoder = CodecUtil.createRawDecoder(conf,
|
||||
ErasureCodeConstants.RS_LEGACY_CODEC_NAME, coderOptions);
|
||||
Assert.assertTrue(decoder instanceof RSRawDecoderLegacy);
|
||||
Assert.assertTrue(decoder instanceof RSLegacyRawDecoder);
|
||||
}
|
||||
|
||||
@Test
|
||||
|
@ -122,10 +122,10 @@ public class TestCodecRawCoderMapping {
|
|||
// should return default raw coder of rs-legacy codec
|
||||
RawErasureEncoder encoder = CodecUtil.createRawEncoder(
|
||||
conf, ErasureCodeConstants.RS_LEGACY_CODEC_NAME, coderOptions);
|
||||
Assert.assertTrue(encoder instanceof RSRawEncoderLegacy);
|
||||
Assert.assertTrue(encoder instanceof RSLegacyRawEncoder);
|
||||
RawErasureDecoder decoder = CodecUtil.createRawDecoder(
|
||||
conf, ErasureCodeConstants.RS_LEGACY_CODEC_NAME, coderOptions);
|
||||
Assert.assertTrue(decoder instanceof RSRawDecoderLegacy);
|
||||
Assert.assertTrue(decoder instanceof RSLegacyRawDecoder);
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
|
@ -60,7 +60,7 @@ public final class RawErasureCoderBenchmark {
|
|||
private static final List<RawErasureCoderFactory> CODER_MAKERS =
|
||||
Collections.unmodifiableList(
|
||||
Arrays.asList(new DummyRawErasureCoderFactory(),
|
||||
new RSRawErasureCoderFactoryLegacy(),
|
||||
new RSLegacyRawErasureCoderFactory(),
|
||||
new RSRawErasureCoderFactory(),
|
||||
new NativeRSRawErasureCoderFactory()));
|
||||
|
||||
|
|
|
@ -29,8 +29,8 @@ import java.nio.ByteBuffer;
|
|||
public class TestDummyRawCoder extends TestRawCoderBase {
|
||||
@Before
|
||||
public void setup() {
|
||||
encoderClass = DummyRawEncoder.class;
|
||||
decoderClass = DummyRawDecoder.class;
|
||||
encoderFactoryClass = DummyRawErasureCoderFactory.class;
|
||||
decoderFactoryClass = DummyRawErasureCoderFactory.class;
|
||||
setAllowDump(false);
|
||||
setChunkSize(baseChunkSize);
|
||||
}
|
||||
|
|
|
@ -30,8 +30,8 @@ public class TestNativeRSRawCoder extends TestRSRawCoderBase {
|
|||
@Before
|
||||
public void setup() {
|
||||
Assume.assumeTrue(ErasureCodeNative.isNativeCodeLoaded());
|
||||
this.encoderClass = NativeRSRawEncoder.class;
|
||||
this.decoderClass = NativeRSRawDecoder.class;
|
||||
this.encoderFactoryClass = NativeRSRawErasureCoderFactory.class;
|
||||
this.decoderFactoryClass = NativeRSRawErasureCoderFactory.class;
|
||||
setAllowDump(true);
|
||||
}
|
||||
|
||||
|
|
|
@ -29,8 +29,8 @@ public class TestNativeXORRawCoder extends TestXORRawCoderBase {
|
|||
@Before
|
||||
public void setup() {
|
||||
Assume.assumeTrue(ErasureCodeNative.isNativeCodeLoaded());
|
||||
this.encoderClass = NativeXORRawEncoder.class;
|
||||
this.decoderClass = NativeXORRawDecoder.class;
|
||||
this.encoderFactoryClass = NativeXORRawErasureCoderFactory.class;
|
||||
this.decoderFactoryClass = NativeXORRawErasureCoderFactory.class;
|
||||
setAllowDump(true);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -22,12 +22,12 @@ import org.junit.Before;
|
|||
/**
|
||||
* Test the legacy raw Reed-solomon coder implemented in Java.
|
||||
*/
|
||||
public class TestRSRawCoderLegacy extends TestRSRawCoderBase {
|
||||
public class TestRSLegacyRawCoder extends TestRSRawCoderBase {
|
||||
|
||||
@Before
|
||||
public void setup() {
|
||||
this.encoderClass = RSRawEncoderLegacy.class;
|
||||
this.decoderClass = RSRawDecoderLegacy.class;
|
||||
this.encoderFactoryClass = RSLegacyRawErasureCoderFactory.class;
|
||||
this.decoderFactoryClass = RSLegacyRawErasureCoderFactory.class;
|
||||
setAllowDump(false); // Change to true to allow verbose dump for debugging
|
||||
}
|
||||
}
|
|
@ -26,8 +26,8 @@ public class TestRSRawCoder extends TestRSRawCoderBase {
|
|||
|
||||
@Before
|
||||
public void setup() {
|
||||
this.encoderClass = RSRawEncoder.class;
|
||||
this.decoderClass = RSRawDecoder.class;
|
||||
this.encoderFactoryClass = RSRawErasureCoderFactory.class;
|
||||
this.decoderFactoryClass = RSRawErasureCoderFactory.class;
|
||||
setAllowDump(false);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -30,8 +30,8 @@ public class TestRSRawCoderInteroperable1 extends TestRSRawCoderBase {
|
|||
public void setup() {
|
||||
Assume.assumeTrue(ErasureCodeNative.isNativeCodeLoaded());
|
||||
|
||||
this.encoderClass = RSRawEncoder.class;
|
||||
this.decoderClass = NativeRSRawDecoder.class;
|
||||
this.encoderFactoryClass = RSRawErasureCoderFactory.class;
|
||||
this.decoderFactoryClass = NativeRSRawErasureCoderFactory.class;
|
||||
setAllowDump(true);
|
||||
}
|
||||
|
||||
|
|
|
@ -30,8 +30,8 @@ public class TestRSRawCoderInteroperable2 extends TestRSRawCoderBase {
|
|||
public void setup() {
|
||||
Assume.assumeTrue(ErasureCodeNative.isNativeCodeLoaded());
|
||||
|
||||
this.encoderClass = NativeRSRawEncoder.class;
|
||||
this.decoderClass = RSRawDecoder.class;
|
||||
this.encoderFactoryClass = NativeRSRawErasureCoderFactory.class;
|
||||
this.decoderFactoryClass = RSRawErasureCoderFactory.class;
|
||||
setAllowDump(true);
|
||||
}
|
||||
|
||||
|
|
|
@ -23,14 +23,12 @@ import org.apache.hadoop.io.erasurecode.TestCoderBase;
|
|||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.lang.reflect.Constructor;
|
||||
|
||||
/**
|
||||
* Raw coder test base with utilities.
|
||||
*/
|
||||
public abstract class TestRawCoderBase extends TestCoderBase {
|
||||
protected Class<? extends RawErasureEncoder> encoderClass;
|
||||
protected Class<? extends RawErasureDecoder> decoderClass;
|
||||
protected Class<? extends RawErasureCoderFactory> encoderFactoryClass;
|
||||
protected Class<? extends RawErasureCoderFactory> decoderFactoryClass;
|
||||
protected RawErasureEncoder encoder;
|
||||
protected RawErasureDecoder decoder;
|
||||
|
||||
|
@ -234,9 +232,8 @@ public abstract class TestRawCoderBase extends TestCoderBase {
|
|||
new ErasureCoderOptions(numDataUnits, numParityUnits,
|
||||
allowChangeInputs, allowDump);
|
||||
try {
|
||||
Constructor<? extends RawErasureEncoder> constructor =
|
||||
encoderClass.getConstructor(ErasureCoderOptions.class);
|
||||
return constructor.newInstance(coderConf);
|
||||
RawErasureCoderFactory factory = encoderFactoryClass.newInstance();
|
||||
return factory.createEncoder(coderConf);
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("Failed to create encoder", e);
|
||||
}
|
||||
|
@ -251,9 +248,8 @@ public abstract class TestRawCoderBase extends TestCoderBase {
|
|||
new ErasureCoderOptions(numDataUnits, numParityUnits,
|
||||
allowChangeInputs, allowDump);
|
||||
try {
|
||||
Constructor<? extends RawErasureDecoder> constructor =
|
||||
decoderClass.getConstructor(ErasureCoderOptions.class);
|
||||
return constructor.newInstance(coderConf);
|
||||
RawErasureCoderFactory factory = encoderFactoryClass.newInstance();
|
||||
return factory.createDecoder(coderConf);
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("Failed to create decoder", e);
|
||||
}
|
||||
|
|
|
@ -26,7 +26,7 @@ public class TestXORRawCoder extends TestXORRawCoderBase {
|
|||
|
||||
@Before
|
||||
public void setup() {
|
||||
this.encoderClass = XORRawEncoder.class;
|
||||
this.decoderClass = XORRawDecoder.class;
|
||||
this.encoderFactoryClass = XORRawErasureCoderFactory.class;
|
||||
this.decoderFactoryClass = XORRawErasureCoderFactory.class;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -29,8 +29,8 @@ public class TestXORRawCoderInteroperable1 extends TestXORRawCoderBase {
|
|||
@Before
|
||||
public void setup() {
|
||||
Assume.assumeTrue(ErasureCodeNative.isNativeCodeLoaded());
|
||||
this.encoderClass = XORRawEncoder.class;
|
||||
this.decoderClass = NativeXORRawDecoder.class;
|
||||
this.encoderFactoryClass = XORRawErasureCoderFactory.class;
|
||||
this.decoderFactoryClass = NativeXORRawErasureCoderFactory.class;
|
||||
setAllowDump(true);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -29,8 +29,8 @@ public class TestXORRawCoderInteroperable2 extends TestXORRawCoderBase {
|
|||
@Before
|
||||
public void setup() {
|
||||
Assume.assumeTrue(ErasureCodeNative.isNativeCodeLoaded());
|
||||
this.encoderClass = NativeXORRawEncoder.class;
|
||||
this.decoderClass = XORRawDecoder.class;
|
||||
this.encoderFactoryClass = NativeXORRawErasureCoderFactory.class;
|
||||
this.decoderFactoryClass = XORRawErasureCoderFactory.class;
|
||||
setAllowDump(true);
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in New Issue