From c3bc083405d84a368bf7281bc109bac2a7c62d0c Mon Sep 17 00:00:00 2001 From: Kai Zheng Date: Wed, 18 Mar 2015 19:21:37 +0800 Subject: [PATCH] HADOOP-11706 Refine a little bit erasure coder API --- .../io/erasurecode/coder/ErasureCoder.java | 4 +++- .../erasurecode/rawcoder/RawErasureCoder.java | 4 +++- .../hadoop/io/erasurecode/TestCoderBase.java | 17 ++++++++++++--- .../coder/TestErasureCoderBase.java | 21 +++---------------- .../erasurecode/rawcoder/TestJRSRawCoder.java | 12 +++++------ .../rawcoder/TestRawCoderBase.java | 2 ++ 6 files changed, 31 insertions(+), 29 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java index 68875c06ad8..c5922f37b15 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.io.erasurecode.coder; +import org.apache.hadoop.conf.Configurable; + /** * An erasure coder to perform encoding or decoding given a group. Generally it * involves calculating necessary internal steps according to codec logic. For @@ -31,7 +33,7 @@ * of multiple coding steps. * */ -public interface ErasureCoder { +public interface ErasureCoder extends Configurable { /** * Initialize with the important parameters for the code. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java index 91a9abfe2aa..9af5b6c0a94 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.io.erasurecode.rawcoder; +import org.apache.hadoop.conf.Configurable; + /** * RawErasureCoder is a common interface for {@link RawErasureEncoder} and * {@link RawErasureDecoder} as both encoder and decoder share some properties. @@ -31,7 +33,7 @@ * low level constructs, since it only takes care of the math calculation with * a group of byte buffers. */ -public interface RawErasureCoder { +public interface RawErasureCoder extends Configurable { /** * Initialize with the important parameters for the code. diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java index 194413a43ac..22fd98d2c4d 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java @@ -17,11 +17,12 @@ */ package org.apache.hadoop.io.erasurecode; +import org.apache.hadoop.conf.Configuration; + import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Random; -import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertTrue; /** @@ -31,6 +32,7 @@ public abstract class TestCoderBase { protected static Random RAND = new Random(); + private Configuration conf; protected int numDataUnits; protected int numParityUnits; protected int chunkSize = 16 * 1024; @@ -49,14 +51,23 @@ public abstract class TestCoderBase { * @param numParityUnits * @param erasedIndexes */ - protected void prepare(int numDataUnits, int numParityUnits, - int[] erasedIndexes) { + protected void prepare(Configuration conf, int numDataUnits, + int numParityUnits, int[] erasedIndexes) { + this.conf = conf; this.numDataUnits = numDataUnits; this.numParityUnits = numParityUnits; this.erasedDataIndexes = erasedIndexes != null ? erasedIndexes : new int[] {0}; } + /** + * Get the conf the test. + * @return configuration + */ + protected Configuration getConf() { + return this.conf; + } + /** * Compare and verify if erased chunks are equal to recovered chunks * @param erasedChunks diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java index d911db9e199..b963a594eb1 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java @@ -17,10 +17,9 @@ */ package org.apache.hadoop.io.erasurecode.coder; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.erasurecode.ECBlock; -import org.apache.hadoop.io.erasurecode.ECChunk; import org.apache.hadoop.io.erasurecode.ECBlockGroup; +import org.apache.hadoop.io.erasurecode.ECChunk; import org.apache.hadoop.io.erasurecode.TestCoderBase; /** @@ -30,7 +29,6 @@ public abstract class TestErasureCoderBase extends TestCoderBase { protected Class encoderClass; protected Class decoderClass; - private Configuration conf; protected int numChunksInBlock = 16; /** @@ -47,19 +45,6 @@ public TestBlock(ECChunk[] chunks) { } } - /** - * Prepare before running the case. - * @param conf - * @param numDataUnits - * @param numParityUnits - * @param erasedIndexes - */ - protected void prepare(Configuration conf, int numDataUnits, - int numParityUnits, int[] erasedIndexes) { - this.conf = conf; - super.prepare(numDataUnits, numParityUnits, erasedIndexes); - } - /** * Generating source data, encoding, recovering and then verifying. * RawErasureCoder mainly uses ECChunk to pass input and output data buffers, @@ -162,7 +147,7 @@ private ErasureEncoder createEncoder() { } encoder.initialize(numDataUnits, numParityUnits, chunkSize); - ((AbstractErasureCoder)encoder).setConf(conf); + encoder.setConf(getConf()); return encoder; } @@ -179,7 +164,7 @@ private ErasureDecoder createDecoder() { } decoder.initialize(numDataUnits, numParityUnits, chunkSize); - ((AbstractErasureCoder)decoder).setConf(conf); + decoder.setConf(getConf()); return decoder; } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestJRSRawCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestJRSRawCoder.java index e54f647cf6c..39e5deb4527 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestJRSRawCoder.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestJRSRawCoder.java @@ -46,37 +46,37 @@ public void setup() { @Test public void testCodingNoDirectBuffer_10x4() { - prepare(10, 4, null); + prepare(null, 10, 4, null); testCoding(false); } @Test public void testCodingDirectBuffer_10x4() { - prepare(10, 4, null); + prepare(null, 10, 4, null); testCoding(true); } @Test public void testCodingDirectBuffer_10x4_erasure_of_2_4() { - prepare(10, 4, new int[] {2, 4}); + prepare(null, 10, 4, new int[] {2, 4}); testCoding(true); } @Test public void testCodingDirectBuffer_10x4_erasing_all() { - prepare(10, 4, new int[] {0, 1, 2, 3}); + prepare(null, 10, 4, new int[] {0, 1, 2, 3}); testCoding(true); } @Test public void testCodingNoDirectBuffer_3x3() { - prepare(3, 3, null); + prepare(null, 3, 3, null); testCoding(false); } @Test public void testCodingDirectBuffer_3x3() { - prepare(3, 3, null); + prepare(null, 3, 3, null); testCoding(true); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java index 890f632370f..b036eed4ced 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java @@ -86,6 +86,7 @@ protected RawErasureEncoder createEncoder() { } encoder.initialize(numDataUnits, numParityUnits, chunkSize); + encoder.setConf(getConf()); return encoder; } @@ -102,6 +103,7 @@ protected RawErasureDecoder createDecoder() { } decoder.initialize(numDataUnits, numParityUnits, chunkSize); + decoder.setConf(getConf()); return decoder; }