HADOOP-12544. Erasure Coding: create dummy raw coder to isolate performance issues in testing. Contributed by Rui Li.
Change-Id: I9856456b59ed881c5ba2acce51e4d9bd01dc6f48
This commit is contained in:
parent
194251c852
commit
3e1745d8e8
|
@ -613,6 +613,9 @@ Trunk (Unreleased)
|
|||
HADOOP-12047. Indicate preference not to affect input buffers during
|
||||
coding in erasure coder. (Kai Zheng via waltersu4549)
|
||||
|
||||
HADOOP-12544. Erasure Coding: create dummy raw coder to isolate performance
|
||||
issues in testing. (Rui Li via zhz)
|
||||
|
||||
Release 2.8.0 - UNRELEASED
|
||||
|
||||
INCOMPATIBLE CHANGES
|
||||
|
|
|
@ -0,0 +1,47 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.io.erasurecode.rawcoder;
|
||||
|
||||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
/**
|
||||
* A dummy raw decoder that does no real computation.
|
||||
* Instead, it just returns zero bytes.
|
||||
* This decoder can be used to isolate the performance issue to HDFS side logic
|
||||
* instead of codec, and is intended for test only.
|
||||
*/
|
||||
@InterfaceAudience.Private
|
||||
public class DummyRawDecoder extends AbstractRawErasureDecoder {
|
||||
public DummyRawDecoder(int numDataUnits, int numParityUnits) {
|
||||
super(numDataUnits, numParityUnits);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void doDecode(ByteBuffer[] inputs, int[] erasedIndexes,
|
||||
ByteBuffer[] outputs) {
|
||||
// Nothing to do. Output buffers have already been reset
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void doDecode(byte[][] inputs, int[] inputOffsets, int dataLen,
|
||||
int[] erasedIndexes, byte[][] outputs, int[] outputOffsets) {
|
||||
// Nothing to do. Output buffers have already been reset
|
||||
}
|
||||
}
|
|
@ -0,0 +1,46 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.io.erasurecode.rawcoder;
|
||||
|
||||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
/**
|
||||
* A dummy raw encoder that does no real computation.
|
||||
* Instead, it just returns zero bytes.
|
||||
* This encoder can be used to isolate the performance issue to HDFS side logic
|
||||
* instead of codec, and is intended for test only.
|
||||
*/
|
||||
@InterfaceAudience.Private
|
||||
public class DummyRawEncoder extends AbstractRawErasureEncoder {
|
||||
public DummyRawEncoder(int numDataUnits, int numParityUnits) {
|
||||
super(numDataUnits, numParityUnits);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void doEncode(ByteBuffer[] inputs, ByteBuffer[] outputs) {
|
||||
// Nothing to do. Output buffers have already been reset
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void doEncode(byte[][] inputs, int[] inputOffsets, int dataLen,
|
||||
byte[][] outputs, int[] outputOffsets) {
|
||||
// Nothing to do. Output buffers have already been reset
|
||||
}
|
||||
}
|
|
@ -0,0 +1,36 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.io.erasurecode.rawcoder;
|
||||
|
||||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
|
||||
/**
|
||||
* A raw erasure coder factory for dummy raw coders.
|
||||
*/
|
||||
@InterfaceAudience.Private
|
||||
public class DummyRawErasureCoderFactory implements RawErasureCoderFactory {
|
||||
@Override
|
||||
public RawErasureEncoder createEncoder(int numDataUnits, int numParityUnits) {
|
||||
return new DummyRawEncoder(numDataUnits, numParityUnits);
|
||||
}
|
||||
|
||||
@Override
|
||||
public RawErasureDecoder createDecoder(int numDataUnits, int numParityUnits) {
|
||||
return new DummyRawDecoder(numDataUnits, numParityUnits);
|
||||
}
|
||||
}
|
|
@ -75,6 +75,10 @@ public abstract class TestCoderBase {
|
|||
this.zeroChunkBytes = new byte[chunkSize]; // With ZERO by default
|
||||
}
|
||||
|
||||
protected byte[] getZeroChunkBytes() {
|
||||
return zeroChunkBytes;
|
||||
}
|
||||
|
||||
protected void prepareBufferAllocator(boolean usingSlicedBuffer) {
|
||||
if (usingSlicedBuffer) {
|
||||
int roughEstimationSpace =
|
||||
|
|
|
@ -0,0 +1,83 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.io.erasurecode.rawcoder;
|
||||
|
||||
import org.apache.hadoop.io.erasurecode.ECChunk;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
/**
|
||||
* Test dummy raw coder.
|
||||
*/
|
||||
public class TestDummyRawCoder extends TestRawCoderBase {
|
||||
@Before
|
||||
public void setup() {
|
||||
encoderClass = DummyRawEncoder.class;
|
||||
decoderClass = DummyRawDecoder.class;
|
||||
setAllowDump(false);
|
||||
setChunkSize(baseChunkSize);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCoding_6x3_erasing_d0_d2() {
|
||||
prepare(null, 6, 3, new int[]{0, 2}, new int[0], false);
|
||||
testCodingDoMixed();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCoding_6x3_erasing_d0_p0() {
|
||||
prepare(null, 6, 3, new int[]{0}, new int[]{0}, false);
|
||||
testCodingDoMixed();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void testCoding(boolean usingDirectBuffer) {
|
||||
this.usingDirectBuffer = usingDirectBuffer;
|
||||
prepareCoders();
|
||||
|
||||
prepareBufferAllocator(true);
|
||||
setAllowChangeInputs(false);
|
||||
|
||||
// Generate data and encode
|
||||
ECChunk[] dataChunks = prepareDataChunksForEncoding();
|
||||
markChunks(dataChunks);
|
||||
ECChunk[] parityChunks = prepareParityChunksForEncoding();
|
||||
encoder.encode(dataChunks, parityChunks);
|
||||
compareAndVerify(parityChunks, getEmptyChunks(parityChunks.length));
|
||||
|
||||
// Decode
|
||||
restoreChunksFromMark(dataChunks);
|
||||
backupAndEraseChunks(dataChunks, parityChunks);
|
||||
ECChunk[] inputChunks = prepareInputChunksForDecoding(
|
||||
dataChunks, parityChunks);
|
||||
ensureOnlyLeastRequiredChunks(inputChunks);
|
||||
ECChunk[] recoveredChunks = prepareOutputChunksForDecoding();
|
||||
decoder.decode(inputChunks, getErasedIndexesForDecoding(), recoveredChunks);
|
||||
compareAndVerify(recoveredChunks, getEmptyChunks(recoveredChunks.length));
|
||||
}
|
||||
|
||||
private ECChunk[] getEmptyChunks(int num) {
|
||||
ECChunk[] chunks = new ECChunk[num];
|
||||
for (int i = 0; i < chunks.length; i++) {
|
||||
chunks[i] = new ECChunk(ByteBuffer.wrap(getZeroChunkBytes()));
|
||||
}
|
||||
return chunks;
|
||||
}
|
||||
}
|
|
@ -30,8 +30,8 @@ import java.lang.reflect.Constructor;
|
|||
public abstract class TestRawCoderBase extends TestCoderBase {
|
||||
protected Class<? extends RawErasureEncoder> encoderClass;
|
||||
protected Class<? extends RawErasureDecoder> decoderClass;
|
||||
private RawErasureEncoder encoder;
|
||||
private RawErasureDecoder decoder;
|
||||
protected RawErasureEncoder encoder;
|
||||
protected RawErasureDecoder decoder;
|
||||
|
||||
/**
|
||||
* Doing twice to test if the coders can be repeatedly reused. This matters
|
||||
|
@ -187,13 +187,13 @@ public abstract class TestRawCoderBase extends TestCoderBase {
|
|||
compareAndVerify(backupChunks, recoveredChunks);
|
||||
}
|
||||
|
||||
private void setAllowChangeInputs(boolean allowChangeInputs) {
|
||||
protected void setAllowChangeInputs(boolean allowChangeInputs) {
|
||||
this.allowChangeInputs = allowChangeInputs;
|
||||
encoder.setCoderOption(CoderOption.ALLOW_CHANGE_INPUTS, allowChangeInputs);
|
||||
decoder.setCoderOption(CoderOption.ALLOW_CHANGE_INPUTS, allowChangeInputs);
|
||||
}
|
||||
|
||||
private void prepareCoders() {
|
||||
protected void prepareCoders() {
|
||||
if (encoder == null) {
|
||||
encoder = createEncoder();
|
||||
}
|
||||
|
@ -203,7 +203,7 @@ public abstract class TestRawCoderBase extends TestCoderBase {
|
|||
}
|
||||
}
|
||||
|
||||
private void ensureOnlyLeastRequiredChunks(ECChunk[] inputChunks) {
|
||||
protected void ensureOnlyLeastRequiredChunks(ECChunk[] inputChunks) {
|
||||
int leastRequiredNum = numDataUnits;
|
||||
int erasedNum = erasedDataIndexes.length + erasedParityIndexes.length;
|
||||
int goodNum = inputChunks.length - erasedNum;
|
||||
|
|
Loading…
Reference in New Issue