Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-12013 Generate fixed data to perform erasure coder test. Contributed by Kai Zheng

This commit is contained in:
Kai Zheng 2015-05-22 23:47:52 +08:00 committed by Zhe Zhang
parent 3d734df24c
commit 4ae32abdf4
4 changed files with 110 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
View File

@ -54,3 +54,5 @@
HADOOP-11938. Enhance ByteBuffer version encode/decode API of raw erasure
coder. (Kai Zheng via Zhe Zhang)
HADOOP-12013. Generate fixed data to perform erasure coder test. (Kai Zheng)

87
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java
View File

@ -52,6 +52,12 @@ public abstract class TestCoderBase {
// may go to different coding implementations.
protected boolean usingDirectBuffer = true;
protected boolean usingFixedData = true;
// Using this the generated data can be repeatable across multiple calls to
// encode(), in order for troubleshooting.
private static int FIXED_DATA_GENERATOR = 0;
protected byte[][] fixedData;
protected int getChunkSize() {
return chunkSize;
}
@ -63,13 +69,17 @@ public abstract class TestCoderBase {
/**
* Prepare before running the case.
* @param conf
* @param numDataUnits
* @param numParityUnits
* @param erasedDataIndexes
* @param erasedParityIndexes
* @param usingFixedData Using fixed or pre-generated data to test instead of
* generating data
*/
protected void prepare(Configuration conf, int numDataUnits,
int numParityUnits, int[] erasedDataIndexes,
int[] erasedParityIndexes) {
int[] erasedParityIndexes, boolean usingFixedData) {
this.conf = conf;
this.numDataUnits = numDataUnits;
this.numParityUnits = numParityUnits;
@ -77,6 +87,38 @@ public abstract class TestCoderBase {
erasedDataIndexes : new int[] {0};
this.erasedParityIndexes = erasedParityIndexes != null ?
erasedParityIndexes : new int[] {0};
this.usingFixedData = usingFixedData;
if (usingFixedData) {
prepareFixedData();
}
}
/**
* Prepare before running the case.
* @param conf
* @param numDataUnits
* @param numParityUnits
* @param erasedDataIndexes
* @param erasedParityIndexes
*/
protected void prepare(Configuration conf, int numDataUnits,
int numParityUnits, int[] erasedDataIndexes,
int[] erasedParityIndexes) {
prepare(conf, numDataUnits, numParityUnits, erasedDataIndexes,
erasedParityIndexes, false);
}
/**
* Prepare before running the case.
* @param numDataUnits
* @param numParityUnits
* @param erasedDataIndexes
* @param erasedParityIndexes
*/
protected void prepare(int numDataUnits, int numParityUnits,
int[] erasedDataIndexes, int[] erasedParityIndexes) {
prepare(null, numDataUnits, numParityUnits, erasedDataIndexes,
erasedParityIndexes, false);
}
/**
@ -278,6 +320,29 @@ public abstract class TestCoderBase {
* @return
*/
protected ECChunk[] prepareDataChunksForEncoding() {
if (usingFixedData) {
ECChunk[] chunks = new ECChunk[numDataUnits];
for (int i = 0; i < chunks.length; i++) {
chunks[i] = makeChunkUsingData(fixedData[i]);
}
return chunks;
}
return generateDataChunks();
}
private ECChunk makeChunkUsingData(byte[] data) {
ECChunk chunk = allocateOutputChunk();
ByteBuffer buffer = chunk.getBuffer();
int pos = buffer.position();
buffer.put(data, 0, chunkSize);
buffer.flip();
buffer.position(pos);
return chunk;
}
private ECChunk[] generateDataChunks() {
ECChunk[] chunks = new ECChunk[numDataUnits];
for (int i = 0; i < chunks.length; i++) {
chunks[i] = generateDataChunk();
@ -286,6 +351,15 @@ public abstract class TestCoderBase {
return chunks;
}
private void prepareFixedData() {
// We may load test data from a resource, or just generate randomly.
// The generated data will be used across subsequent encode/decode calls.
this.fixedData = new byte[numDataUnits][];
for (int i = 0; i < numDataUnits; i++) {
fixedData[i] = generateFixedData(baseChunkSize * 2);
}
}
/**
* Generate data chunk by making random data.
* @return
@ -319,6 +393,17 @@ public abstract class TestCoderBase {
return buffer;
}
protected byte[] generateFixedData(int len) {
byte[] buffer = new byte[len];
for (int i = 0; i < buffer.length; i++) {
buffer[i] = (byte) FIXED_DATA_GENERATOR++;
if (FIXED_DATA_GENERATOR == 256) {
FIXED_DATA_GENERATOR = 0;
}
}
return buffer;
}
/**
* Prepare parity chunks for encoding, each chunk for each parity unit.
* @return

8
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoder.java
View File

@ -80,7 +80,13 @@ public class TestRSRawCoder extends TestRSRawCoderBase {
@Test
public void testCodingDirectBuffer_10x4_erasure_of_d2_d4_p0() {
prepare(null, 10, 4, new int[] {2, 4}, new int[] {0});
prepare(null, 10, 4, new int[]{2, 4}, new int[]{0});
testCoding(true);
}
@Test
public void testCodingDirectBuffer_usingFixedData_10x4_erasure_of_d2_d4_p0() {
prepare(null, 10, 4, new int[] {2, 4}, new int[] {0}, true);
testCoding(true);
}

21
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoderBase.java
View File

@ -17,12 +17,7 @@
*/
package org.apache.hadoop.io.erasurecode.rawcoder;
import org.apache.hadoop.io.erasurecode.ECChunk;
import org.apache.hadoop.io.erasurecode.rawcoder.util.RSUtil;
import org.junit.Before;
import org.junit.Test;
import java.nio.ByteBuffer;
/**
* Test base for raw Reed-solomon coders.
@ -32,6 +27,8 @@ public abstract class TestRSRawCoderBase extends TestRawCoderBase {
private static int symbolSize = 0;
private static int symbolMax = 0;
private static int RS_FIXED_DATA_GENERATOR = 0;
static {
symbolSize = (int) Math.round(Math.log(
RSUtil.GF.getFieldSize()) / Math.log(2));
@ -41,9 +38,21 @@ public abstract class TestRSRawCoderBase extends TestRawCoderBase {
@Override
protected byte[] generateData(int len) {
byte[] buffer = new byte[len];
for (int i = 0; i < len; i++) {
for (int i = 0; i < buffer.length; i++) {
buffer[i] = (byte) RAND.nextInt(symbolMax);
}
return buffer;
}
@Override
protected byte[] generateFixedData(int len) {
byte[] buffer = new byte[len];
for (int i = 0; i < buffer.length; i++) {
buffer[i] = (byte) RS_FIXED_DATA_GENERATOR++;
if (RS_FIXED_DATA_GENERATOR == symbolMax) {
RS_FIXED_DATA_GENERATOR = 0;
}
}
return buffer;
}
}