Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-14479. Erasurecode testcase failures with native enabled. Contributed by Sammi Chen

This commit is contained in:
Kai Zheng 2017-06-29 11:26:36 +08:00
parent 20ba86d66a
commit ea1da39b19
4 changed files with 15 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncodingStep.java
View File

@ -62,6 +62,6 @@ public ECBlock[] getOutputBlocks() {
@Override @Override
public void finish() { public void finish() {
rawEncoder.release(); // do nothing
} }
} }

5
hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_decoder.c
View File

@ -31,7 +31,7 @@
typedef struct _XOREncoder { typedef struct _XOREncoder {
IsalCoder isalCoder; IsalCoder isalCoder;
unsigned char* inputs[MMAX]; unsigned char* inputs[MMAX];
unsigned char* outputs[1]; unsigned char* outputs[KMAX];
} XORDecoder; } XORDecoder;
JNIEXPORT void JNICALL JNIEXPORT void JNICALL
@ -58,8 +58,7 @@ Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeXORRawDecoder_decodeImpl(
numParityUnits = ((IsalCoder*)xorDecoder)->numParityUnits; numParityUnits = ((IsalCoder*)xorDecoder)->numParityUnits;
chunkSize = (int)dataLen; chunkSize = (int)dataLen;
getInputs(env, inputs, inputOffsets, xorDecoder->inputs, getInputs(env, inputs, inputOffsets, xorDecoder->inputs, numDataUnits);
numDataUnits + numParityUnits);
getOutputs(env, outputs, outputOffsets, xorDecoder->outputs, numParityUnits); getOutputs(env, outputs, outputOffsets, xorDecoder->outputs, numParityUnits);
for (i = 0; i < numDataUnits + numParityUnits; i++) { for (i = 0; i < numDataUnits + numParityUnits; i++) {

2
hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_encoder.c
View File

@ -31,7 +31,7 @@
typedef struct _XOREncoder { typedef struct _XOREncoder {
IsalCoder isalCoder; IsalCoder isalCoder;
unsigned char* inputs[MMAX]; unsigned char* inputs[MMAX];
unsigned char* outputs[1]; unsigned char* outputs[KMAX];
} XOREncoder; } XOREncoder;
JNIEXPORT void JNICALL JNIEXPORT void JNICALL

13
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java
View File

@ -18,11 +18,13 @@
package org.apache.hadoop.io.erasurecode; package org.apache.hadoop.io.erasurecode;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawEncoder;
import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawDecoder;
import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory; import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory;
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawDecoder;
import org.apache.hadoop.io.erasurecode.rawcoder.RSLegacyRawDecoder; import org.apache.hadoop.io.erasurecode.rawcoder.RSLegacyRawDecoder;
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawEncoder;
import org.apache.hadoop.io.erasurecode.rawcoder.RSLegacyRawEncoder; import org.apache.hadoop.io.erasurecode.rawcoder.RSLegacyRawEncoder;
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawDecoder;
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawEncoder;
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory; import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
@ -55,10 +57,15 @@ public void testRSDefaultRawCoder() {
// should return default raw coder of rs codec // should return default raw coder of rs codec
RawErasureEncoder encoder = CodecUtil.createRawEncoder( RawErasureEncoder encoder = CodecUtil.createRawEncoder(
conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions); conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions);
Assert.assertTrue(encoder instanceof RSRawEncoder);
RawErasureDecoder decoder = CodecUtil.createRawDecoder( RawErasureDecoder decoder = CodecUtil.createRawDecoder(
conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions); conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions);
if (ErasureCodeNative.isNativeCodeLoaded()) {
Assert.assertTrue(encoder instanceof NativeRSRawEncoder);
Assert.assertTrue(decoder instanceof NativeRSRawDecoder);
} else {
Assert.assertTrue(encoder instanceof RSRawEncoder);
Assert.assertTrue(decoder instanceof RSRawDecoder); Assert.assertTrue(decoder instanceof RSRawDecoder);
}
// should return default raw coder of rs-legacy codec // should return default raw coder of rs-legacy codec
encoder = CodecUtil.createRawEncoder(conf, encoder = CodecUtil.createRawEncoder(conf,