HDFS-11823. Extend TestDFSStripedIutputStream/TestDFSStripedOutputStream with a random EC policy. Contributed by Takanobu Asanuma.

This commit is contained in:
Jing Zhao 2017-05-24 11:14:19 -07:00
parent dcf4559ebd
commit 1c8dd6d3d1
7 changed files with 63 additions and 121 deletions

View File

@ -49,6 +49,7 @@ import java.util.Arrays;
import java.util.Collections; import java.util.Collections;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Random;
import java.util.Set; import java.util.Set;
import java.util.concurrent.TimeoutException; import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
@ -568,4 +569,15 @@ public class StripedFileTestUtil {
public static ErasureCodingPolicy getDefaultECPolicy() { public static ErasureCodingPolicy getDefaultECPolicy() {
return SystemErasureCodingPolicies.getPolicies().get(0); return SystemErasureCodingPolicies.getPolicies().get(0);
} }
/**
* Get non-default Erasure Coding Policy randomly.
* @return ErasureCodingPolicy
*/
public static ErasureCodingPolicy getRandomNonDefaultECPolicy() {
Random rand = new Random();
List<ErasureCodingPolicy> policies = SystemErasureCodingPolicies
.getPolicies();
return policies.get(1 + rand.nextInt(policies.size() - 1));
}
} }

View File

@ -17,18 +17,29 @@
*/ */
package org.apache.hadoop.hdfs; package org.apache.hadoop.hdfs;
import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
/** /**
* This tests read operation of DFS striped file with RS-10-4-64k * This tests read operation of DFS striped file with a random erasure code
* erasure code policy. * policy except for the default policy.
*/ */
public class TestDFSRSDefault10x4StripedInputStream extends public class TestDFSStripedInputStreamWithRandomECPolicy extends
TestDFSStripedInputStream { TestDFSStripedInputStream {
private static final Log LOG = LogFactory.getLog(
TestDFSStripedInputStreamWithRandomECPolicy.class.getName());
private ErasureCodingPolicy ecPolicy;
public TestDFSStripedInputStreamWithRandomECPolicy() {
ecPolicy = StripedFileTestUtil.getRandomNonDefaultECPolicy();
LOG.info(ecPolicy);
}
@Override
public ErasureCodingPolicy getEcPolicy() { public ErasureCodingPolicy getEcPolicy() {
return SystemErasureCodingPolicies.getByID( return ecPolicy;
SystemErasureCodingPolicies.RS_10_4_POLICY_ID);
} }
} }

View File

@ -17,19 +17,29 @@
*/ */
package org.apache.hadoop.hdfs; package org.apache.hadoop.hdfs;
import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
/** /**
* This tests write operation of DFS striped file with RS-10-4-64k * This tests write operation of DFS striped file with a random erasure code
* erasure code policy under Datanode failure conditions. * policy except for the default policy under Datanode failure conditions.
*/ */
public class TestDFSRSDefault10x4StripedOutputStreamWithFailure public class TestDFSStripedOutputStreamWithFailureWithRandomECPolicy extends
extends TestDFSStripedOutputStreamWithFailure { TestDFSStripedOutputStreamWithFailure {
private static final Log LOG = LogFactory.getLog(
TestDFSStripedOutputStreamWithRandomECPolicy.class.getName());
private ErasureCodingPolicy ecPolicy;
public TestDFSStripedOutputStreamWithFailureWithRandomECPolicy() {
ecPolicy = StripedFileTestUtil.getRandomNonDefaultECPolicy();
LOG.info(ecPolicy);
}
@Override @Override
public ErasureCodingPolicy getEcPolicy() { public ErasureCodingPolicy getEcPolicy() {
return SystemErasureCodingPolicies.getByID( return ecPolicy;
SystemErasureCodingPolicies.RS_10_4_POLICY_ID);
} }
} }

View File

@ -17,19 +17,29 @@
*/ */
package org.apache.hadoop.hdfs; package org.apache.hadoop.hdfs;
import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
/** /**
* This tests write operation of DFS striped file with RS-10-4-64k * This tests write operation of DFS striped file with a random erasure code
* erasure code policy. * policy except for the default policy.
*/ */
public class TestDFSRSDefault10x4StripedOutputStream public class TestDFSStripedOutputStreamWithRandomECPolicy extends
extends TestDFSStripedOutputStream { TestDFSStripedOutputStream {
private static final Log LOG = LogFactory.getLog(
TestDFSStripedOutputStreamWithRandomECPolicy.class.getName());
private ErasureCodingPolicy ecPolicy;
public TestDFSStripedOutputStreamWithRandomECPolicy() {
ecPolicy = StripedFileTestUtil.getRandomNonDefaultECPolicy();
LOG.info(ecPolicy);
}
@Override @Override
public ErasureCodingPolicy getEcPolicy() { public ErasureCodingPolicy getEcPolicy() {
return SystemErasureCodingPolicies.getByID( return ecPolicy;
SystemErasureCodingPolicies.RS_10_4_POLICY_ID);
} }
} }

View File

@ -1,32 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs;
import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
/**
* This tests read operation of DFS striped file with XOR-2-1-64k erasure code
* policy.
*/
public class TestDFSXORStripedInputStream extends TestDFSStripedInputStream{
public ErasureCodingPolicy getEcPolicy() {
return SystemErasureCodingPolicies.getByID(
SystemErasureCodingPolicies.XOR_2_1_POLICY_ID);
}
}

View File

@ -1,34 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs;
import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
/**
* This tests write operation of DFS striped file with XOR-2-1-64k erasure code
* policy.
*/
public class TestDFSXORStripedOutputStream extends TestDFSStripedOutputStream{
@Override
public ErasureCodingPolicy getEcPolicy() {
return SystemErasureCodingPolicies.getByID(
SystemErasureCodingPolicies.XOR_2_1_POLICY_ID);
}
}

View File

@ -1,35 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs;
import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
/**
* This tests write operation of DFS striped file with XOR-2-1-64k erasure code
* policy when there is data node failure.
*/
public class TestDFSXORStripedOutputStreamWithFailure
extends TestDFSStripedOutputStreamWithFailure{
@Override
public ErasureCodingPolicy getEcPolicy() {
return SystemErasureCodingPolicies.getByID(
SystemErasureCodingPolicies.XOR_2_1_POLICY_ID);
}
}