HDFS-11823. Extend TestDFSStripedIutputStream/TestDFSStripedOutputStream with a random EC policy. Contributed by Takanobu Asanuma.
This commit is contained in:
parent
dcf4559ebd
commit
1c8dd6d3d1
|
@ -49,6 +49,7 @@ import java.util.Arrays;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.Random;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.concurrent.TimeoutException;
|
import java.util.concurrent.TimeoutException;
|
||||||
import java.util.concurrent.atomic.AtomicInteger;
|
import java.util.concurrent.atomic.AtomicInteger;
|
||||||
|
@ -568,4 +569,15 @@ public class StripedFileTestUtil {
|
||||||
public static ErasureCodingPolicy getDefaultECPolicy() {
|
public static ErasureCodingPolicy getDefaultECPolicy() {
|
||||||
return SystemErasureCodingPolicies.getPolicies().get(0);
|
return SystemErasureCodingPolicies.getPolicies().get(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get non-default Erasure Coding Policy randomly.
|
||||||
|
* @return ErasureCodingPolicy
|
||||||
|
*/
|
||||||
|
public static ErasureCodingPolicy getRandomNonDefaultECPolicy() {
|
||||||
|
Random rand = new Random();
|
||||||
|
List<ErasureCodingPolicy> policies = SystemErasureCodingPolicies
|
||||||
|
.getPolicies();
|
||||||
|
return policies.get(1 + rand.nextInt(policies.size() - 1));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -17,18 +17,29 @@
|
||||||
*/
|
*/
|
||||||
package org.apache.hadoop.hdfs;
|
package org.apache.hadoop.hdfs;
|
||||||
|
|
||||||
import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
|
import org.apache.commons.logging.Log;
|
||||||
|
import org.apache.commons.logging.LogFactory;
|
||||||
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
|
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* This tests read operation of DFS striped file with RS-10-4-64k
|
* This tests read operation of DFS striped file with a random erasure code
|
||||||
* erasure code policy.
|
* policy except for the default policy.
|
||||||
*/
|
*/
|
||||||
public class TestDFSRSDefault10x4StripedInputStream extends
|
public class TestDFSStripedInputStreamWithRandomECPolicy extends
|
||||||
TestDFSStripedInputStream {
|
TestDFSStripedInputStream {
|
||||||
|
|
||||||
|
private static final Log LOG = LogFactory.getLog(
|
||||||
|
TestDFSStripedInputStreamWithRandomECPolicy.class.getName());
|
||||||
|
|
||||||
|
private ErasureCodingPolicy ecPolicy;
|
||||||
|
|
||||||
|
public TestDFSStripedInputStreamWithRandomECPolicy() {
|
||||||
|
ecPolicy = StripedFileTestUtil.getRandomNonDefaultECPolicy();
|
||||||
|
LOG.info(ecPolicy);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
public ErasureCodingPolicy getEcPolicy() {
|
public ErasureCodingPolicy getEcPolicy() {
|
||||||
return SystemErasureCodingPolicies.getByID(
|
return ecPolicy;
|
||||||
SystemErasureCodingPolicies.RS_10_4_POLICY_ID);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
|
@ -17,19 +17,29 @@
|
||||||
*/
|
*/
|
||||||
package org.apache.hadoop.hdfs;
|
package org.apache.hadoop.hdfs;
|
||||||
|
|
||||||
import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
|
import org.apache.commons.logging.Log;
|
||||||
|
import org.apache.commons.logging.LogFactory;
|
||||||
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
|
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* This tests write operation of DFS striped file with RS-10-4-64k
|
* This tests write operation of DFS striped file with a random erasure code
|
||||||
* erasure code policy under Datanode failure conditions.
|
* policy except for the default policy under Datanode failure conditions.
|
||||||
*/
|
*/
|
||||||
public class TestDFSRSDefault10x4StripedOutputStreamWithFailure
|
public class TestDFSStripedOutputStreamWithFailureWithRandomECPolicy extends
|
||||||
extends TestDFSStripedOutputStreamWithFailure {
|
TestDFSStripedOutputStreamWithFailure {
|
||||||
|
|
||||||
|
private static final Log LOG = LogFactory.getLog(
|
||||||
|
TestDFSStripedOutputStreamWithRandomECPolicy.class.getName());
|
||||||
|
|
||||||
|
private ErasureCodingPolicy ecPolicy;
|
||||||
|
|
||||||
|
public TestDFSStripedOutputStreamWithFailureWithRandomECPolicy() {
|
||||||
|
ecPolicy = StripedFileTestUtil.getRandomNonDefaultECPolicy();
|
||||||
|
LOG.info(ecPolicy);
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public ErasureCodingPolicy getEcPolicy() {
|
public ErasureCodingPolicy getEcPolicy() {
|
||||||
return SystemErasureCodingPolicies.getByID(
|
return ecPolicy;
|
||||||
SystemErasureCodingPolicies.RS_10_4_POLICY_ID);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
|
@ -17,19 +17,29 @@
|
||||||
*/
|
*/
|
||||||
package org.apache.hadoop.hdfs;
|
package org.apache.hadoop.hdfs;
|
||||||
|
|
||||||
import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
|
import org.apache.commons.logging.Log;
|
||||||
|
import org.apache.commons.logging.LogFactory;
|
||||||
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
|
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* This tests write operation of DFS striped file with RS-10-4-64k
|
* This tests write operation of DFS striped file with a random erasure code
|
||||||
* erasure code policy.
|
* policy except for the default policy.
|
||||||
*/
|
*/
|
||||||
public class TestDFSRSDefault10x4StripedOutputStream
|
public class TestDFSStripedOutputStreamWithRandomECPolicy extends
|
||||||
extends TestDFSStripedOutputStream {
|
TestDFSStripedOutputStream {
|
||||||
|
|
||||||
|
private static final Log LOG = LogFactory.getLog(
|
||||||
|
TestDFSStripedOutputStreamWithRandomECPolicy.class.getName());
|
||||||
|
|
||||||
|
private ErasureCodingPolicy ecPolicy;
|
||||||
|
|
||||||
|
public TestDFSStripedOutputStreamWithRandomECPolicy() {
|
||||||
|
ecPolicy = StripedFileTestUtil.getRandomNonDefaultECPolicy();
|
||||||
|
LOG.info(ecPolicy);
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public ErasureCodingPolicy getEcPolicy() {
|
public ErasureCodingPolicy getEcPolicy() {
|
||||||
return SystemErasureCodingPolicies.getByID(
|
return ecPolicy;
|
||||||
SystemErasureCodingPolicies.RS_10_4_POLICY_ID);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
|
@ -1,32 +0,0 @@
|
||||||
/**
|
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one
|
|
||||||
* or more contributor license agreements. See the NOTICE file
|
|
||||||
* distributed with this work for additional information
|
|
||||||
* regarding copyright ownership. The ASF licenses this file
|
|
||||||
* to you under the Apache License, Version 2.0 (the
|
|
||||||
* "License"); you may not use this file except in compliance
|
|
||||||
* with the License. You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
package org.apache.hadoop.hdfs;
|
|
||||||
import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
|
|
||||||
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* This tests read operation of DFS striped file with XOR-2-1-64k erasure code
|
|
||||||
* policy.
|
|
||||||
*/
|
|
||||||
public class TestDFSXORStripedInputStream extends TestDFSStripedInputStream{
|
|
||||||
|
|
||||||
public ErasureCodingPolicy getEcPolicy() {
|
|
||||||
return SystemErasureCodingPolicies.getByID(
|
|
||||||
SystemErasureCodingPolicies.XOR_2_1_POLICY_ID);
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,34 +0,0 @@
|
||||||
/**
|
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one
|
|
||||||
* or more contributor license agreements. See the NOTICE file
|
|
||||||
* distributed with this work for additional information
|
|
||||||
* regarding copyright ownership. The ASF licenses this file
|
|
||||||
* to you under the Apache License, Version 2.0 (the
|
|
||||||
* "License"); you may not use this file except in compliance
|
|
||||||
* with the License. You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
package org.apache.hadoop.hdfs;
|
|
||||||
|
|
||||||
import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
|
|
||||||
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* This tests write operation of DFS striped file with XOR-2-1-64k erasure code
|
|
||||||
* policy.
|
|
||||||
*/
|
|
||||||
public class TestDFSXORStripedOutputStream extends TestDFSStripedOutputStream{
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ErasureCodingPolicy getEcPolicy() {
|
|
||||||
return SystemErasureCodingPolicies.getByID(
|
|
||||||
SystemErasureCodingPolicies.XOR_2_1_POLICY_ID);
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,35 +0,0 @@
|
||||||
/**
|
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one
|
|
||||||
* or more contributor license agreements. See the NOTICE file
|
|
||||||
* distributed with this work for additional information
|
|
||||||
* regarding copyright ownership. The ASF licenses this file
|
|
||||||
* to you under the Apache License, Version 2.0 (the
|
|
||||||
* "License"); you may not use this file except in compliance
|
|
||||||
* with the License. You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
package org.apache.hadoop.hdfs;
|
|
||||||
|
|
||||||
import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
|
|
||||||
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* This tests write operation of DFS striped file with XOR-2-1-64k erasure code
|
|
||||||
* policy when there is data node failure.
|
|
||||||
*/
|
|
||||||
public class TestDFSXORStripedOutputStreamWithFailure
|
|
||||||
extends TestDFSStripedOutputStreamWithFailure{
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public ErasureCodingPolicy getEcPolicy() {
|
|
||||||
return SystemErasureCodingPolicies.getByID(
|
|
||||||
SystemErasureCodingPolicies.XOR_2_1_POLICY_ID);
|
|
||||||
}
|
|
||||||
}
|
|
Loading…
Reference in New Issue