HADOOP-11740. Combine erasure encoder and decoder interfaces. Contributed by Zhe Zhang.

This commit is contained in:
Zhe Zhang 2015-04-03 15:22:50 -07:00 committed by Zhe Zhang
parent 1af8c14862
commit e54a74b566
7 changed files with 41 additions and 99 deletions

View File

@ -23,13 +23,12 @@ import org.apache.hadoop.io.erasurecode.ECBlockGroup;
/**
* An abstract erasure decoder that's to be inherited by new decoders.
*
* It implements the {@link ErasureDecoder} interface.
* It implements the {@link ErasureCoder} interface.
*/
public abstract class AbstractErasureDecoder extends AbstractErasureCoder
implements ErasureDecoder {
public abstract class AbstractErasureDecoder extends AbstractErasureCoder {
@Override
public ErasureCodingStep decode(ECBlockGroup blockGroup) {
public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup) {
// We may have more than this when considering complicate cases. HADOOP-11550
return prepareDecodingStep(blockGroup);
}

View File

@ -23,13 +23,12 @@ import org.apache.hadoop.io.erasurecode.ECBlockGroup;
/**
* An abstract erasure encoder that's to be inherited by new encoders.
*
* It implements the {@link ErasureEncoder} interface.
* It implements the {@link ErasureCoder} interface.
*/
public abstract class AbstractErasureEncoder extends AbstractErasureCoder
implements ErasureEncoder {
public abstract class AbstractErasureEncoder extends AbstractErasureCoder {
@Override
public ErasureCodingStep encode(ECBlockGroup blockGroup) {
public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup) {
// We may have more than this when considering complicate cases. HADOOP-11550
return prepareEncodingStep(blockGroup);
}

View File

@ -18,6 +18,7 @@
package org.apache.hadoop.io.erasurecode.coder;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.io.erasurecode.ECBlockGroup;
/**
* An erasure coder to perform encoding or decoding given a group. Generally it
@ -63,6 +64,17 @@ public interface ErasureCoder extends Configurable {
*/
public int getChunkSize();
/**
* Calculate the encoding or decoding steps given a block blockGroup.
*
* Note, currently only one coding step is supported. Will support complex
* cases of multiple coding steps.
*
* @param blockGroup the erasure coding block group containing all necessary
* information for codec calculation
*/
public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup);
/**
* Tell if native or off-heap buffer is preferred or not. It's for callers to
* decide how to allocate coding chunk buffers, either on heap or off heap.

View File

@ -1,41 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.io.erasurecode.coder;
import org.apache.hadoop.io.erasurecode.ECBlockGroup;
/**
* Erasure decoder interface to perform decoding given a block group.
*
* It extends {@link ErasureCoder}.
*/
public interface ErasureDecoder extends ErasureCoder {
/**
* Perform the decoding given a blockGroup. By default it will try the best to
* attempt to recover all the missing blocks according to the codec logic.
*
* Note, currently only one coding step is supported. Will support complex
* cases of multiple coding steps.
*
* @param blockGroup
*/
public ErasureCodingStep decode(ECBlockGroup blockGroup);
}

View File

@ -1,39 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.io.erasurecode.coder;
import org.apache.hadoop.io.erasurecode.ECBlockGroup;
/**
* Erasure encoder interface to perform encoding given a block group.
*
* It extends {@link ErasureCoder}.
*/
public interface ErasureEncoder extends ErasureCoder {
/**
* Calculate the encoding steps given a block blockGroup.
*
* Note, currently only one coding step is supported. Will support complex
* cases of multiple coding steps.
*
* @param blockGroup
*/
public ErasureCodingStep encode(ECBlockGroup blockGroup);
}

View File

@ -26,8 +26,8 @@ import org.apache.hadoop.io.erasurecode.TestCoderBase;
* Erasure coder test base with utilities.
*/
public abstract class TestErasureCoderBase extends TestCoderBase {
protected Class<? extends ErasureEncoder> encoderClass;
protected Class<? extends ErasureDecoder> decoderClass;
protected Class<? extends ErasureCoder> encoderClass;
protected Class<? extends ErasureCoder> decoderClass;
protected int numChunksInBlock = 16;
@ -55,7 +55,7 @@ public abstract class TestErasureCoderBase extends TestCoderBase {
protected void testCoding(boolean usingDirectBuffer) {
this.usingDirectBuffer = usingDirectBuffer;
ErasureEncoder encoder = createEncoder();
ErasureCoder encoder = createEncoder();
// Generate data and encode
ECBlockGroup blockGroup = prepareBlockGroupForEncoding();
@ -68,7 +68,7 @@ public abstract class TestErasureCoderBase extends TestCoderBase {
ErasureCodingStep codingStep;
try {
codingStep = encoder.encode(blockGroup);
codingStep = encoder.calculateCoding(blockGroup);
performCodingStep(codingStep);
} finally {
encoder.release();
@ -78,9 +78,9 @@ public abstract class TestErasureCoderBase extends TestCoderBase {
//Decode
blockGroup = new ECBlockGroup(clonedDataBlocks, blockGroup.getParityBlocks());
ErasureDecoder decoder = createDecoder();
ErasureCoder decoder = createDecoder();
try {
codingStep = decoder.decode(blockGroup);
codingStep = decoder.calculateCoding(blockGroup);
performCodingStep(codingStep);
} finally {
decoder.release();
@ -138,8 +138,8 @@ public abstract class TestErasureCoderBase extends TestCoderBase {
* Create erasure encoder for test.
* @return
*/
private ErasureEncoder createEncoder() {
ErasureEncoder encoder;
private ErasureCoder createEncoder() {
ErasureCoder encoder;
try {
encoder = encoderClass.newInstance();
} catch (Exception e) {
@ -155,8 +155,8 @@ public abstract class TestErasureCoderBase extends TestCoderBase {
* Create the erasure decoder for the test.
* @return
*/
private ErasureDecoder createDecoder() {
ErasureDecoder decoder;
private ErasureCoder createDecoder() {
ErasureCoder decoder;
try {
decoder = decoderClass.newInstance();
} catch (Exception e) {

View File

@ -43,4 +43,16 @@
blocks in NameNode (Jing Zhao)
HDFS-8005. Erasure Coding: simplify striped block recovery work computation
and add tests (Jing Zhao)
and add tests (Jing Zhao)
HDFS-7617. Add unit tests for editlog transactions for EC
(Hui Zheng via Zhe Zhang)
HADOOP-11782. Correct two thrown messages in ECSchema class
(Xinwei Qin via Kai Zheng)
HDFS-7839. Erasure coding: implement facilities in NameNode to create and
manage EC zones (Zhe Zhang)
HADOOP-11740. Combine erasure encoder and decoder interfaces (Zhe Zhang)