From d88dca844a8437397959f28dadf1294e6db31df9 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Tue, 25 Oct 2016 16:53:53 -0700 Subject: [PATCH] HADOOP-11798. Native raw erasure coder in XOR codes. Contributed by SammiChen. --- hadoop-common-project/hadoop-common/pom.xml | 4 + .../hadoop-common/src/CMakeLists.txt | 4 +- .../rawcoder/NativeXORRawDecoder.java | 59 +++++++++++++ .../rawcoder/NativeXORRawEncoder.java | 60 ++++++++++++++ .../NativeXORRawErasureCoderFactory.java | 39 +++++++++ .../src/main/native/native.vcxproj | 6 +- .../hadoop/io/erasurecode/jni_xor_decoder.c | 80 ++++++++++++++++++ .../hadoop/io/erasurecode/jni_xor_encoder.c | 82 +++++++++++++++++++ .../rawcoder/TestNativeXORRawCoder.java | 36 ++++++++ .../erasurecode/rawcoder/TestXORRawCoder.java | 38 +-------- .../rawcoder/TestXORRawCoderBase.java | 59 +++++++++++++ .../TestXORRawCoderInteroperable1.java | 36 ++++++++ .../TestXORRawCoderInteroperable2.java | 37 +++++++++ 13 files changed, 501 insertions(+), 39 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawDecoder.java create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawEncoder.java create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawErasureCoderFactory.java create mode 100644 hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_decoder.c create mode 100644 hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_encoder.c create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestNativeXORRawCoder.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestXORRawCoderBase.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestXORRawCoderInteroperable1.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestXORRawCoderInteroperable2.java diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 54d1cddb8a4..fd9b7cd5cae 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -640,6 +640,8 @@ org.apache.hadoop.io.erasurecode.ErasureCodeNative org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawEncoder org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawDecoder + org.apache.hadoop.io.erasurecode.rawcoder.NativeXORRawEncoder + org.apache.hadoop.io.erasurecode.rawcoder.NativeXORRawDecoder org.apache.hadoop.crypto.OpensslCipher org.apache.hadoop.crypto.random.OpensslSecureRandom org.apache.hadoop.util.NativeCrc32 @@ -780,6 +782,8 @@ org.apache.hadoop.io.erasurecode.ErasureCodeNative org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawEncoder org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawDecoder + org.apache.hadoop.io.erasurecode.rawcoder.NativeXORRawEncoder + org.apache.hadoop.io.erasurecode.rawcoder.NativeXORRawDecoder org.apache.hadoop.crypto.OpensslCipher org.apache.hadoop.crypto.random.OpensslSecureRandom org.apache.hadoop.util.NativeCrc32 diff --git a/hadoop-common-project/hadoop-common/src/CMakeLists.txt b/hadoop-common-project/hadoop-common/src/CMakeLists.txt index 8026eb54cf4..8317a461620 100644 --- a/hadoop-common-project/hadoop-common/src/CMakeLists.txt +++ b/hadoop-common-project/hadoop-common/src/CMakeLists.txt @@ -113,7 +113,9 @@ if (ISAL_LIBRARY) ${SRC}/io/erasurecode/jni_erasure_code_native.c ${SRC}/io/erasurecode/jni_common.c ${SRC}/io/erasurecode/jni_rs_encoder.c - ${SRC}/io/erasurecode/jni_rs_decoder.c) + ${SRC}/io/erasurecode/jni_rs_decoder.c + ${SRC}/io/erasurecode/jni_xor_encoder.c + ${SRC}/io/erasurecode/jni_xor_decoder.c) add_executable(erasure_code_test ${SRC}/io/erasurecode/isal_load.c diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawDecoder.java new file mode 100644 index 00000000000..b6b16733126 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawDecoder.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.erasurecode.rawcoder; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.io.erasurecode.ErasureCodeNative; +import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; + +import java.nio.ByteBuffer; + +/** + * A XOR raw decoder using Intel ISA-L library. + */ +@InterfaceAudience.Private +public class NativeXORRawDecoder extends AbstractNativeRawDecoder { + + static { + ErasureCodeNative.checkNativeCodeLoaded(); + } + + public NativeXORRawDecoder(ErasureCoderOptions coderOptions) { + super(coderOptions); + initImpl(coderOptions.getNumDataUnits(), coderOptions.getNumParityUnits()); + } + + @Override + protected void performDecodeImpl(ByteBuffer[] inputs, int[] inputOffsets, + int dataLen, int[] erased, ByteBuffer[] outputs, int[] outputOffsets) { + decodeImpl(inputs, inputOffsets, dataLen, erased, outputs, outputOffsets); + } + + @Override + public void release() { + destroyImpl(); + } + + private native void initImpl(int numDataUnits, int numParityUnits); + + private native void decodeImpl( + ByteBuffer[] inputs, int[] inputOffsets, int dataLen, int[] erased, + ByteBuffer[] outputs, int[] outputOffsets); + + private native void destroyImpl(); +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawEncoder.java new file mode 100644 index 00000000000..9b4b44912c4 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawEncoder.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.erasurecode.rawcoder; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.io.erasurecode.ErasureCodeNative; +import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; + +import java.nio.ByteBuffer; + +/** + * A XOR raw encoder using Intel ISA-L library. + */ +@InterfaceAudience.Private +public class NativeXORRawEncoder extends AbstractNativeRawEncoder { + + static { + ErasureCodeNative.checkNativeCodeLoaded(); + } + + public NativeXORRawEncoder(ErasureCoderOptions coderOptions) { + super(coderOptions); + initImpl(coderOptions.getNumDataUnits(), coderOptions.getNumParityUnits()); + } + + @Override + protected void performEncodeImpl( + ByteBuffer[] inputs, int[] inputOffsets, int dataLen, + ByteBuffer[] outputs, int[] outputOffsets) { + encodeImpl(inputs, inputOffsets, dataLen, outputs, outputOffsets); + } + + @Override + public void release() { + destroyImpl(); + } + + private native void initImpl(int numDataUnits, int numParityUnits); + + private native void encodeImpl(ByteBuffer[] inputs, int[] inputOffsets, + int dataLen, ByteBuffer[] outputs, + int[] outputOffsets); + + private native void destroyImpl(); +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawErasureCoderFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawErasureCoderFactory.java new file mode 100644 index 00000000000..66b3f7830a9 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawErasureCoderFactory.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.erasurecode.rawcoder; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.io.erasurecode.ErasureCoderOptions; + +/** + * A raw coder factory for xor coder in native using Intel ISA-L library. + */ + +@InterfaceAudience.Private +public class NativeXORRawErasureCoderFactory implements RawErasureCoderFactory { + + @Override + public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) { + return new NativeXORRawEncoder(coderOptions); + } + + @Override + public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) { + return new NativeXORRawDecoder(coderOptions); + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj b/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj index 2274c413319..11190698f81 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj +++ b/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj @@ -156,14 +156,14 @@ /D HADOOP_SNAPPY_LIBRARY=L\"snappy.dll\" - /D HADOOP_ISAL_LIBRARY=\"isa-l.dll\" + /D HADOOP_ISAL_LIBRARY=L\"isa-l.dll\" src\org\apache\hadoop\io\nativeio;%(AdditionalIncludeDirectories) - /D HADOOP_ISAL_LIBRARY=\"isa-l.dll\" + /D HADOOP_ISAL_LIBRARY=L\"isa-l.dll\" @@ -173,6 +173,8 @@ + + diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_decoder.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_decoder.c new file mode 100644 index 00000000000..ff0d3c5a912 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_decoder.c @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +#include "org_apache_hadoop.h" +#include "erasure_code.h" +#include "gf_util.h" +#include "jni_common.h" +#include "org_apache_hadoop_io_erasurecode_rawcoder_NativeXORRawDecoder.h" + +typedef struct _XOREncoder { + IsalCoder isalCoder; + unsigned char* inputs[MMAX]; + unsigned char* outputs[1]; +} XORDecoder; + +JNIEXPORT void JNICALL +Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeXORRawDecoder_initImpl( + JNIEnv *env, jobject thiz, jint numDataUnits, jint numParityUnits) { + XORDecoder* xorDecoder = + (XORDecoder*)malloc(sizeof(XORDecoder)); + memset(xorDecoder, 0, sizeof(*xorDecoder)); + initCoder(&xorDecoder->isalCoder, numDataUnits, numParityUnits); + + setCoder(env, thiz, &xorDecoder->isalCoder); +} + +JNIEXPORT void JNICALL +Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeXORRawDecoder_decodeImpl( + JNIEnv *env, jobject thiz, jobjectArray inputs, jintArray inputOffsets, + jint dataLen, jintArray erasedIndexes, jobjectArray outputs, + jintArray outputOffsets) { + int i, j, numDataUnits, numParityUnits, chunkSize; + XORDecoder* xorDecoder; + + xorDecoder = (XORDecoder*)getCoder(env, thiz); + numDataUnits = ((IsalCoder*)xorDecoder)->numDataUnits; + numParityUnits = ((IsalCoder*)xorDecoder)->numParityUnits; + chunkSize = (int)dataLen; + + getInputs(env, inputs, inputOffsets, xorDecoder->inputs, + numDataUnits + numParityUnits); + getOutputs(env, outputs, outputOffsets, xorDecoder->outputs, numParityUnits); + + for (i = 0; i < numDataUnits + numParityUnits; i++) { + if (xorDecoder->inputs[i] == NULL) { + continue; + } + for (j = 0; j < chunkSize; j++) { + xorDecoder->outputs[0][j] ^= xorDecoder->inputs[i][j]; + } + } +} + +JNIEXPORT void JNICALL +Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeXORRawDecoder_destroyImpl + (JNIEnv *env, jobject thiz){ + XORDecoder* xorDecoder = (XORDecoder*)getCoder(env, thiz); + free(xorDecoder); +} diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_encoder.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_encoder.c new file mode 100644 index 00000000000..3cfa01ca8e8 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_xor_encoder.c @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +#include "org_apache_hadoop.h" +#include "erasure_code.h" +#include "gf_util.h" +#include "jni_common.h" +#include "org_apache_hadoop_io_erasurecode_rawcoder_NativeXORRawEncoder.h" + +typedef struct _XOREncoder { + IsalCoder isalCoder; + unsigned char* inputs[MMAX]; + unsigned char* outputs[1]; +} XOREncoder; + +JNIEXPORT void JNICALL +Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeXORRawEncoder_initImpl + (JNIEnv *env, jobject thiz, jint numDataUnits, jint numParityUnits) { + XOREncoder* xorEncoder = + (XOREncoder*)malloc(sizeof(XOREncoder)); + memset(xorEncoder, 0, sizeof(*xorEncoder)); + initCoder(&xorEncoder->isalCoder, numDataUnits, numParityUnits); + + setCoder(env, thiz, &xorEncoder->isalCoder); +} + +JNIEXPORT void JNICALL +Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeXORRawEncoder_encodeImpl( + JNIEnv *env, jobject thiz, jobjectArray inputs, jintArray inputOffsets, + jint dataLen, jobjectArray outputs, jintArray outputOffsets) { + + int i, j, numDataUnits, numParityUnits, chunkSize; + XOREncoder* xorEncoder; + + xorEncoder = (XOREncoder*)getCoder(env, thiz); + numDataUnits = ((IsalCoder*)xorEncoder)->numDataUnits; + numParityUnits = ((IsalCoder*)xorEncoder)->numParityUnits; + chunkSize = (int)dataLen; + + getInputs(env, inputs, inputOffsets, xorEncoder->inputs, numDataUnits); + getOutputs(env, outputs, outputOffsets, xorEncoder->outputs, numParityUnits); + + // Get the first buffer's data. + for (j = 0; j < chunkSize; j++) { + xorEncoder->outputs[0][j] = xorEncoder->inputs[0][j]; + } + + // XOR with everything else. + for (i = 1; i < numDataUnits; i++) { + for (j = 0; j < chunkSize; j++) { + xorEncoder->outputs[0][j] ^= xorEncoder->inputs[i][j]; + } + } +} + +JNIEXPORT void JNICALL +Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeXORRawEncoder_destroyImpl + (JNIEnv *env, jobject thiz) { + XOREncoder* xorEncoder = (XOREncoder*)getCoder(env, thiz); + free(xorEncoder); +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestNativeXORRawCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestNativeXORRawCoder.java new file mode 100644 index 00000000000..ba4b7b4c299 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestNativeXORRawCoder.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.erasurecode.rawcoder; + +import org.apache.hadoop.io.erasurecode.ErasureCodeNative; +import org.junit.Assume; +import org.junit.Before; + +/** + * Test NativeXOR encoding and decoding. + */ +public class TestNativeXORRawCoder extends TestXORRawCoderBase { + + @Before + public void setup() { + Assume.assumeTrue(ErasureCodeNative.isNativeCodeLoaded()); + this.encoderClass = NativeXORRawEncoder.class; + this.decoderClass = NativeXORRawDecoder.class; + setAllowDump(true); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestXORRawCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestXORRawCoder.java index 48463ad1fd3..aae3a048c84 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestXORRawCoder.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestXORRawCoder.java @@ -18,49 +18,15 @@ package org.apache.hadoop.io.erasurecode.rawcoder; import org.junit.Before; -import org.junit.Test; /** - * Test XOR encoding and decoding. + * Test pure Java XOR encoding and decoding. */ -public class TestXORRawCoder extends TestRawCoderBase { +public class TestXORRawCoder extends TestXORRawCoderBase { @Before public void setup() { this.encoderClass = XORRawEncoder.class; this.decoderClass = XORRawDecoder.class; } - - @Test - public void testCoding_10x1_erasing_d0() { - prepare(null, 10, 1, new int[] {0}, new int[0]); - testCodingDoMixAndTwice(); - } - - @Test - public void testCoding_10x1_erasing_p0() { - prepare(null, 10, 1, new int[0], new int[] {0}); - testCodingDoMixAndTwice(); - } - - @Test - public void testCoding_10x1_erasing_d5() { - prepare(null, 10, 1, new int[]{5}, new int[0]); - testCodingDoMixAndTwice(); - } - - @Test - public void testCodingNegative_10x1_erasing_too_many() { - prepare(null, 10, 1, new int[]{2}, new int[]{0}); - testCodingWithErasingTooMany(); - } - - @Test - public void testCodingNegative_10x1_erasing_d5() { - prepare(null, 10, 1, new int[]{5}, new int[0]); - testCodingWithBadInput(true); - testCodingWithBadOutput(false); - testCodingWithBadInput(true); - testCodingWithBadOutput(false); - } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestXORRawCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestXORRawCoderBase.java new file mode 100644 index 00000000000..27de379f0b0 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestXORRawCoderBase.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.erasurecode.rawcoder; + +import org.junit.Test; + +/** + * Test base for raw XOR coders. + */ +public abstract class TestXORRawCoderBase extends TestRawCoderBase { + + @Test + public void testCoding_10x1_erasing_d0() { + prepare(null, 10, 1, new int[] {0}, new int[0]); + testCodingDoMixAndTwice(); + } + + @Test + public void testCoding_10x1_erasing_p0() { + prepare(null, 10, 1, new int[0], new int[] {0}); + testCodingDoMixAndTwice(); + } + + @Test + public void testCoding_10x1_erasing_d5() { + prepare(null, 10, 1, new int[]{5}, new int[0]); + testCodingDoMixAndTwice(); + } + + @Test + public void testCodingNegative_10x1_erasing_too_many() { + prepare(null, 10, 1, new int[]{2}, new int[]{0}); + testCodingWithErasingTooMany(); + } + + @Test + public void testCodingNegative_10x1_erasing_d5() { + prepare(null, 10, 1, new int[]{5}, new int[0]); + testCodingWithBadInput(true); + testCodingWithBadOutput(false); + testCodingWithBadInput(true); + testCodingWithBadOutput(false); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestXORRawCoderInteroperable1.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestXORRawCoderInteroperable1.java new file mode 100644 index 00000000000..c5a809c8218 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestXORRawCoderInteroperable1.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.erasurecode.rawcoder; + +import org.apache.hadoop.io.erasurecode.ErasureCodeNative; +import org.junit.Assume; +import org.junit.Before; + +/** + * Test raw XOR coder implemented in Java. + */ +public class TestXORRawCoderInteroperable1 extends TestXORRawCoderBase { + + @Before + public void setup() { + Assume.assumeTrue(ErasureCodeNative.isNativeCodeLoaded()); + this.encoderClass = XORRawEncoder.class; + this.decoderClass = NativeXORRawDecoder.class; + setAllowDump(true); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestXORRawCoderInteroperable2.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestXORRawCoderInteroperable2.java new file mode 100644 index 00000000000..5345eb69ae8 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestXORRawCoderInteroperable2.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.erasurecode.rawcoder; + +import org.apache.hadoop.io.erasurecode.ErasureCodeNative; +import org.junit.Assume; +import org.junit.Before; + +/** + * Test raw XOR coder implemented in Java. + */ +public class TestXORRawCoderInteroperable2 extends TestXORRawCoderBase { + + @Before + public void setup() { + Assume.assumeTrue(ErasureCodeNative.isNativeCodeLoaded()); + this.encoderClass = NativeXORRawEncoder.class; + this.decoderClass = XORRawDecoder.class; + setAllowDump(true); + } + +}