HDFS-12587. Use Parameterized tests in TestBlockInfoStriped and TestLowRedundancyBlockQueues to test all EC policies. Contributed by Takanobu Asanuma.
(cherry picked from commit 3b8dbc2cb7
)
This commit is contained in:
parent
8f4bfb3ed5
commit
c0aa249a1d
|
@ -46,6 +46,7 @@ import java.io.IOException;
|
||||||
import java.nio.ByteBuffer;
|
import java.nio.ByteBuffer;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
|
import java.util.Collection;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
@ -591,4 +592,18 @@ public class StripedFileTestUtil {
|
||||||
.getPolicies();
|
.getPolicies();
|
||||||
return policies.get(1 + rand.nextInt(policies.size() - 1));
|
return policies.get(1 + rand.nextInt(policies.size() - 1));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get all Erasure Coding Policies for Parameterized tests.
|
||||||
|
* @return Collection<Object[]>
|
||||||
|
*/
|
||||||
|
public static Collection<Object[]> getECPolicies() {
|
||||||
|
ArrayList<Object[]> params = new ArrayList<>();
|
||||||
|
List<ErasureCodingPolicy> policies =
|
||||||
|
SystemErasureCodingPolicies.getPolicies();
|
||||||
|
for (ErasureCodingPolicy policy: policies) {
|
||||||
|
params.add(new Object[]{policy});
|
||||||
|
}
|
||||||
|
return params;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -25,29 +25,42 @@ import org.junit.Assert;
|
||||||
import org.junit.Rule;
|
import org.junit.Rule;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
import org.junit.rules.Timeout;
|
import org.junit.rules.Timeout;
|
||||||
|
import org.junit.runner.RunWith;
|
||||||
|
import org.junit.runners.Parameterized;
|
||||||
import org.mockito.internal.util.reflection.Whitebox;
|
import org.mockito.internal.util.reflection.Whitebox;
|
||||||
|
|
||||||
import java.io.DataOutput;
|
import java.io.DataOutput;
|
||||||
import java.io.DataOutputStream;
|
import java.io.DataOutputStream;
|
||||||
import java.io.ByteArrayOutputStream;
|
import java.io.ByteArrayOutputStream;
|
||||||
import java.nio.ByteBuffer;
|
import java.nio.ByteBuffer;
|
||||||
|
import java.util.Collection;
|
||||||
|
|
||||||
import static org.junit.Assert.assertArrayEquals;
|
import static org.junit.Assert.assertArrayEquals;
|
||||||
import static org.junit.Assert.assertEquals;
|
import static org.junit.Assert.assertEquals;
|
||||||
import static org.junit.Assert.fail;
|
import static org.junit.Assert.fail;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Test {@link BlockInfoStriped}
|
* Test {@link BlockInfoStriped}.
|
||||||
*/
|
*/
|
||||||
|
@RunWith(Parameterized.class)
|
||||||
public class TestBlockInfoStriped {
|
public class TestBlockInfoStriped {
|
||||||
private static final long BASE_ID = -1600;
|
private static final long BASE_ID = -1600;
|
||||||
private final Block baseBlock = new Block(BASE_ID);
|
private final Block baseBlock = new Block(BASE_ID);
|
||||||
private final ErasureCodingPolicy testECPolicy
|
private final ErasureCodingPolicy testECPolicy;
|
||||||
= StripedFileTestUtil.getDefaultECPolicy();
|
private final int totalBlocks;
|
||||||
private final int totalBlocks = testECPolicy.getNumDataUnits() +
|
private final BlockInfoStriped info;
|
||||||
testECPolicy.getNumParityUnits();
|
|
||||||
private final BlockInfoStriped info = new BlockInfoStriped(baseBlock,
|
public TestBlockInfoStriped(ErasureCodingPolicy policy) {
|
||||||
testECPolicy);
|
testECPolicy = policy;
|
||||||
|
totalBlocks = testECPolicy.getNumDataUnits()
|
||||||
|
+ testECPolicy.getNumParityUnits();
|
||||||
|
info = new BlockInfoStriped(baseBlock, testECPolicy);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Parameterized.Parameters(name = "{index}: {0}")
|
||||||
|
public static Collection<Object[]> policies() {
|
||||||
|
return StripedFileTestUtil.getECPolicies();
|
||||||
|
}
|
||||||
|
|
||||||
private Block[] createReportedBlocks(int num) {
|
private Block[] createReportedBlocks(int num) {
|
||||||
Block[] blocks = new Block[num];
|
Block[] blocks = new Block[num];
|
||||||
|
@ -61,7 +74,7 @@ public class TestBlockInfoStriped {
|
||||||
public Timeout globalTimeout = new Timeout(300000);
|
public Timeout globalTimeout = new Timeout(300000);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Test adding storage and reported block
|
* Test adding storage and reported block.
|
||||||
*/
|
*/
|
||||||
@Test
|
@Test
|
||||||
public void testAddStorage() {
|
public void testAddStorage() {
|
||||||
|
@ -108,8 +121,8 @@ public class TestBlockInfoStriped {
|
||||||
}
|
}
|
||||||
|
|
||||||
// the same block is reported from another storage
|
// the same block is reported from another storage
|
||||||
DatanodeStorageInfo[] storageInfos2 = DFSTestUtil.createDatanodeStorageInfos(
|
DatanodeStorageInfo[] storageInfos2 =
|
||||||
totalBlocks * 2);
|
DFSTestUtil.createDatanodeStorageInfos(totalBlocks * 2);
|
||||||
// only add the second half of info2
|
// only add the second half of info2
|
||||||
for (i = totalBlocks; i < storageInfos2.length; i++) {
|
for (i = totalBlocks; i < storageInfos2.length; i++) {
|
||||||
info.addStorage(storageInfos2[i], blocks[i % totalBlocks]);
|
info.addStorage(storageInfos2[i], blocks[i % totalBlocks]);
|
||||||
|
|
|
@ -18,22 +18,37 @@
|
||||||
|
|
||||||
package org.apache.hadoop.hdfs.server.blockmanagement;
|
package org.apache.hadoop.hdfs.server.blockmanagement;
|
||||||
|
|
||||||
|
import java.util.Collection;
|
||||||
import java.util.Iterator;
|
import java.util.Iterator;
|
||||||
|
|
||||||
import org.apache.hadoop.hdfs.StripedFileTestUtil;
|
import org.apache.hadoop.hdfs.StripedFileTestUtil;
|
||||||
import org.apache.hadoop.hdfs.protocol.Block;
|
import org.apache.hadoop.hdfs.protocol.Block;
|
||||||
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
|
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
import org.junit.runner.RunWith;
|
||||||
|
import org.junit.runners.Parameterized;
|
||||||
|
|
||||||
import static org.junit.Assert.assertEquals;
|
import static org.junit.Assert.assertEquals;
|
||||||
import static org.junit.Assert.assertTrue;
|
import static org.junit.Assert.assertTrue;
|
||||||
import static org.junit.Assert.assertFalse;
|
import static org.junit.Assert.assertFalse;
|
||||||
import static org.junit.Assert.fail;
|
import static org.junit.Assert.fail;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test {@link LowRedundancyBlocks}.
|
||||||
|
*/
|
||||||
|
@RunWith(Parameterized.class)
|
||||||
public class TestLowRedundancyBlockQueues {
|
public class TestLowRedundancyBlockQueues {
|
||||||
|
|
||||||
private final ErasureCodingPolicy ecPolicy =
|
private final ErasureCodingPolicy ecPolicy;
|
||||||
StripedFileTestUtil.getDefaultECPolicy();
|
|
||||||
|
public TestLowRedundancyBlockQueues(ErasureCodingPolicy policy) {
|
||||||
|
ecPolicy = policy;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Parameterized.Parameters(name = "{index}: {0}")
|
||||||
|
public static Collection<Object[]> policies() {
|
||||||
|
return StripedFileTestUtil.getECPolicies();
|
||||||
|
}
|
||||||
|
|
||||||
private BlockInfo genBlockInfo(long id) {
|
private BlockInfo genBlockInfo(long id) {
|
||||||
return new BlockInfoContiguous(new Block(id), (short) 3);
|
return new BlockInfoContiguous(new Block(id), (short) 3);
|
||||||
|
|
Loading…
Reference in New Issue