HADOOP-12544. Erasure Coding: create dummy raw coder to isolate performance issues in testing. Contributed by Rui Li.

Change-Id: I9856456b59ed881c5ba2acce51e4d9bd01dc6f48
This commit is contained in:
Zhe Zhang 2015-11-03 22:26:27 -08:00
parent 194251c852
commit 3e1745d8e8
7 changed files with 224 additions and 5 deletions

View File

@ -613,6 +613,9 @@ Trunk (Unreleased)
HADOOP-12047. Indicate preference not to affect input buffers during
coding in erasure coder. (Kai Zheng via waltersu4549)
HADOOP-12544. Erasure Coding: create dummy raw coder to isolate performance
issues in testing. (Rui Li via zhz)
Release 2.8.0 - UNRELEASED
INCOMPATIBLE CHANGES

View File

@ -0,0 +1,47 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.io.erasurecode.rawcoder;
import org.apache.hadoop.classification.InterfaceAudience;
import java.nio.ByteBuffer;
/**
* A dummy raw decoder that does no real computation.
* Instead, it just returns zero bytes.
* This decoder can be used to isolate the performance issue to HDFS side logic
* instead of codec, and is intended for test only.
*/
@InterfaceAudience.Private
public class DummyRawDecoder extends AbstractRawErasureDecoder {
public DummyRawDecoder(int numDataUnits, int numParityUnits) {
super(numDataUnits, numParityUnits);
}
@Override
protected void doDecode(ByteBuffer[] inputs, int[] erasedIndexes,
ByteBuffer[] outputs) {
// Nothing to do. Output buffers have already been reset
}
@Override
protected void doDecode(byte[][] inputs, int[] inputOffsets, int dataLen,
int[] erasedIndexes, byte[][] outputs, int[] outputOffsets) {
// Nothing to do. Output buffers have already been reset
}
}

View File

@ -0,0 +1,46 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.io.erasurecode.rawcoder;
import org.apache.hadoop.classification.InterfaceAudience;
import java.nio.ByteBuffer;
/**
* A dummy raw encoder that does no real computation.
* Instead, it just returns zero bytes.
* This encoder can be used to isolate the performance issue to HDFS side logic
* instead of codec, and is intended for test only.
*/
@InterfaceAudience.Private
public class DummyRawEncoder extends AbstractRawErasureEncoder {
public DummyRawEncoder(int numDataUnits, int numParityUnits) {
super(numDataUnits, numParityUnits);
}
@Override
protected void doEncode(ByteBuffer[] inputs, ByteBuffer[] outputs) {
// Nothing to do. Output buffers have already been reset
}
@Override
protected void doEncode(byte[][] inputs, int[] inputOffsets, int dataLen,
byte[][] outputs, int[] outputOffsets) {
// Nothing to do. Output buffers have already been reset
}
}

View File

@ -0,0 +1,36 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.io.erasurecode.rawcoder;
import org.apache.hadoop.classification.InterfaceAudience;
/**
* A raw erasure coder factory for dummy raw coders.
*/
@InterfaceAudience.Private
public class DummyRawErasureCoderFactory implements RawErasureCoderFactory {
@Override
public RawErasureEncoder createEncoder(int numDataUnits, int numParityUnits) {
return new DummyRawEncoder(numDataUnits, numParityUnits);
}
@Override
public RawErasureDecoder createDecoder(int numDataUnits, int numParityUnits) {
return new DummyRawDecoder(numDataUnits, numParityUnits);
}
}

View File

@ -75,6 +75,10 @@ public abstract class TestCoderBase {
this.zeroChunkBytes = new byte[chunkSize]; // With ZERO by default
}
protected byte[] getZeroChunkBytes() {
return zeroChunkBytes;
}
protected void prepareBufferAllocator(boolean usingSlicedBuffer) {
if (usingSlicedBuffer) {
int roughEstimationSpace =

View File

@ -0,0 +1,83 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.io.erasurecode.rawcoder;
import org.apache.hadoop.io.erasurecode.ECChunk;
import org.junit.Before;
import org.junit.Test;
import java.nio.ByteBuffer;
/**
* Test dummy raw coder.
*/
public class TestDummyRawCoder extends TestRawCoderBase {
@Before
public void setup() {
encoderClass = DummyRawEncoder.class;
decoderClass = DummyRawDecoder.class;
setAllowDump(false);
setChunkSize(baseChunkSize);
}
@Test
public void testCoding_6x3_erasing_d0_d2() {
prepare(null, 6, 3, new int[]{0, 2}, new int[0], false);
testCodingDoMixed();
}
@Test
public void testCoding_6x3_erasing_d0_p0() {
prepare(null, 6, 3, new int[]{0}, new int[]{0}, false);
testCodingDoMixed();
}
@Override
protected void testCoding(boolean usingDirectBuffer) {
this.usingDirectBuffer = usingDirectBuffer;
prepareCoders();
prepareBufferAllocator(true);
setAllowChangeInputs(false);
// Generate data and encode
ECChunk[] dataChunks = prepareDataChunksForEncoding();
markChunks(dataChunks);
ECChunk[] parityChunks = prepareParityChunksForEncoding();
encoder.encode(dataChunks, parityChunks);
compareAndVerify(parityChunks, getEmptyChunks(parityChunks.length));
// Decode
restoreChunksFromMark(dataChunks);
backupAndEraseChunks(dataChunks, parityChunks);
ECChunk[] inputChunks = prepareInputChunksForDecoding(
dataChunks, parityChunks);
ensureOnlyLeastRequiredChunks(inputChunks);
ECChunk[] recoveredChunks = prepareOutputChunksForDecoding();
decoder.decode(inputChunks, getErasedIndexesForDecoding(), recoveredChunks);
compareAndVerify(recoveredChunks, getEmptyChunks(recoveredChunks.length));
}
private ECChunk[] getEmptyChunks(int num) {
ECChunk[] chunks = new ECChunk[num];
for (int i = 0; i < chunks.length; i++) {
chunks[i] = new ECChunk(ByteBuffer.wrap(getZeroChunkBytes()));
}
return chunks;
}
}

View File

@ -30,8 +30,8 @@ import java.lang.reflect.Constructor;
public abstract class TestRawCoderBase extends TestCoderBase {
protected Class<? extends RawErasureEncoder> encoderClass;
protected Class<? extends RawErasureDecoder> decoderClass;
private RawErasureEncoder encoder;
private RawErasureDecoder decoder;
protected RawErasureEncoder encoder;
protected RawErasureDecoder decoder;
/**
* Doing twice to test if the coders can be repeatedly reused. This matters
@ -187,13 +187,13 @@ public abstract class TestRawCoderBase extends TestCoderBase {
compareAndVerify(backupChunks, recoveredChunks);
}
private void setAllowChangeInputs(boolean allowChangeInputs) {
protected void setAllowChangeInputs(boolean allowChangeInputs) {
this.allowChangeInputs = allowChangeInputs;
encoder.setCoderOption(CoderOption.ALLOW_CHANGE_INPUTS, allowChangeInputs);
decoder.setCoderOption(CoderOption.ALLOW_CHANGE_INPUTS, allowChangeInputs);
}
private void prepareCoders() {
protected void prepareCoders() {
if (encoder == null) {
encoder = createEncoder();
}
@ -203,7 +203,7 @@ public abstract class TestRawCoderBase extends TestCoderBase {
}
}
private void ensureOnlyLeastRequiredChunks(ECChunk[] inputChunks) {
protected void ensureOnlyLeastRequiredChunks(ECChunk[] inputChunks) {
int leastRequiredNum = numDataUnits;
int erasedNum = erasedDataIndexes.length + erasedParityIndexes.length;
int goodNum = inputChunks.length - erasedNum;