From ea1da39b1908de7739eaecded64074355f0fd5d0 Mon Sep 17 00:00:00 2001 From: Kai Zheng Date: Thu, 29 Jun 2017 11:26:36 +0800 Subject: [PATCH] HADOOP-14479. Erasurecode testcase failures with native enabled. Contributed by Sammi Chen --- .../io/erasurecode/coder/ErasureEncodingStep.java | 2 +- .../hadoop/io/erasurecode/jni_xor_decoder.c | 5 ++--- .../hadoop/io/erasurecode/jni_xor_encoder.c | 2 +- .../io/erasurecode/TestCodecRawCoderMapping.java | 15 +++++++++++---- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncodingStep.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncodingStep.java index f0b2be8f58b..df4ed4b4063 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncodingStep.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncodingStep.java @@ -62,6 +62,6 @@ public ECBlock[] getOutputBlocks() { @Override public void finish() { - rawEncoder.release(); + // do nothing } } diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_decoder.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_decoder.c index ff0d3c5a912..13f66e2d8ed 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_decoder.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_decoder.c @@ -31,7 +31,7 @@ typedef struct _XOREncoder { IsalCoder isalCoder; unsigned char* inputs[MMAX]; - unsigned char* outputs[1]; + unsigned char* outputs[KMAX]; } XORDecoder; JNIEXPORT void JNICALL @@ -58,8 +58,7 @@ Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeXORRawDecoder_decodeImpl( numParityUnits = ((IsalCoder*)xorDecoder)->numParityUnits; chunkSize = (int)dataLen; - getInputs(env, inputs, inputOffsets, xorDecoder->inputs, - numDataUnits + numParityUnits); + getInputs(env, inputs, inputOffsets, xorDecoder->inputs, numDataUnits); getOutputs(env, outputs, outputOffsets, xorDecoder->outputs, numParityUnits); for (i = 0; i < numDataUnits + numParityUnits; i++) { diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_encoder.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_encoder.c index 3cfa01ca8e8..6efb5fb9515 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_encoder.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_encoder.c @@ -31,7 +31,7 @@ typedef struct _XOREncoder { IsalCoder isalCoder; unsigned char* inputs[MMAX]; - unsigned char* outputs[1]; + unsigned char* outputs[KMAX]; } XOREncoder; JNIEXPORT void JNICALL diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java index 4a6fbd41f55..c7874dc77a2 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java @@ -18,11 +18,13 @@ package org.apache.hadoop.io.erasurecode; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawEncoder; +import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawDecoder; import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory; -import org.apache.hadoop.io.erasurecode.rawcoder.RSRawDecoder; import org.apache.hadoop.io.erasurecode.rawcoder.RSLegacyRawDecoder; -import org.apache.hadoop.io.erasurecode.rawcoder.RSRawEncoder; import org.apache.hadoop.io.erasurecode.rawcoder.RSLegacyRawEncoder; +import org.apache.hadoop.io.erasurecode.rawcoder.RSRawDecoder; +import org.apache.hadoop.io.erasurecode.rawcoder.RSRawEncoder; import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder; @@ -55,10 +57,15 @@ public void testRSDefaultRawCoder() { // should return default raw coder of rs codec RawErasureEncoder encoder = CodecUtil.createRawEncoder( conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions); - Assert.assertTrue(encoder instanceof RSRawEncoder); RawErasureDecoder decoder = CodecUtil.createRawDecoder( conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions); - Assert.assertTrue(decoder instanceof RSRawDecoder); + if (ErasureCodeNative.isNativeCodeLoaded()) { + Assert.assertTrue(encoder instanceof NativeRSRawEncoder); + Assert.assertTrue(decoder instanceof NativeRSRawDecoder); + } else { + Assert.assertTrue(encoder instanceof RSRawEncoder); + Assert.assertTrue(decoder instanceof RSRawDecoder); + } // should return default raw coder of rs-legacy codec encoder = CodecUtil.createRawEncoder(conf,