HDFS-12587. Use Parameterized tests in TestBlockInfoStriped and TestLowRedundancyBlockQueues to test all EC policies. Contributed by Takanobu Asanuma.

(cherry picked from commit 3b8dbc2cb7)
This commit is contained in:
Xiao Chen 2018-03-13 10:12:52 -07:00
parent 8f4bfb3ed5
commit c0aa249a1d
3 changed files with 55 additions and 12 deletions

View File

@ -46,6 +46,7 @@ import java.io.IOException;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
@ -591,4 +592,18 @@ public class StripedFileTestUtil {
.getPolicies(); .getPolicies();
return policies.get(1 + rand.nextInt(policies.size() - 1)); return policies.get(1 + rand.nextInt(policies.size() - 1));
} }
/**
* Get all Erasure Coding Policies for Parameterized tests.
* @return Collection<Object[]>
*/
public static Collection<Object[]> getECPolicies() {
ArrayList<Object[]> params = new ArrayList<>();
List<ErasureCodingPolicy> policies =
SystemErasureCodingPolicies.getPolicies();
for (ErasureCodingPolicy policy: policies) {
params.add(new Object[]{policy});
}
return params;
}
} }

View File

@ -25,29 +25,42 @@ import org.junit.Assert;
import org.junit.Rule; import org.junit.Rule;
import org.junit.Test; import org.junit.Test;
import org.junit.rules.Timeout; import org.junit.rules.Timeout;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.mockito.internal.util.reflection.Whitebox; import org.mockito.internal.util.reflection.Whitebox;
import java.io.DataOutput; import java.io.DataOutput;
import java.io.DataOutputStream; import java.io.DataOutputStream;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.util.Collection;
import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail; import static org.junit.Assert.fail;
/** /**
* Test {@link BlockInfoStriped} * Test {@link BlockInfoStriped}.
*/ */
@RunWith(Parameterized.class)
public class TestBlockInfoStriped { public class TestBlockInfoStriped {
private static final long BASE_ID = -1600; private static final long BASE_ID = -1600;
private final Block baseBlock = new Block(BASE_ID); private final Block baseBlock = new Block(BASE_ID);
private final ErasureCodingPolicy testECPolicy private final ErasureCodingPolicy testECPolicy;
= StripedFileTestUtil.getDefaultECPolicy(); private final int totalBlocks;
private final int totalBlocks = testECPolicy.getNumDataUnits() + private final BlockInfoStriped info;
testECPolicy.getNumParityUnits();
private final BlockInfoStriped info = new BlockInfoStriped(baseBlock, public TestBlockInfoStriped(ErasureCodingPolicy policy) {
testECPolicy); testECPolicy = policy;
totalBlocks = testECPolicy.getNumDataUnits()
+ testECPolicy.getNumParityUnits();
info = new BlockInfoStriped(baseBlock, testECPolicy);
}
@Parameterized.Parameters(name = "{index}: {0}")
public static Collection<Object[]> policies() {
return StripedFileTestUtil.getECPolicies();
}
private Block[] createReportedBlocks(int num) { private Block[] createReportedBlocks(int num) {
Block[] blocks = new Block[num]; Block[] blocks = new Block[num];
@ -61,7 +74,7 @@ public class TestBlockInfoStriped {
public Timeout globalTimeout = new Timeout(300000); public Timeout globalTimeout = new Timeout(300000);
/** /**
* Test adding storage and reported block * Test adding storage and reported block.
*/ */
@Test @Test
public void testAddStorage() { public void testAddStorage() {
@ -108,8 +121,8 @@ public class TestBlockInfoStriped {
} }
// the same block is reported from another storage // the same block is reported from another storage
DatanodeStorageInfo[] storageInfos2 = DFSTestUtil.createDatanodeStorageInfos( DatanodeStorageInfo[] storageInfos2 =
totalBlocks * 2); DFSTestUtil.createDatanodeStorageInfos(totalBlocks * 2);
// only add the second half of info2 // only add the second half of info2
for (i = totalBlocks; i < storageInfos2.length; i++) { for (i = totalBlocks; i < storageInfos2.length; i++) {
info.addStorage(storageInfos2[i], blocks[i % totalBlocks]); info.addStorage(storageInfos2[i], blocks[i % totalBlocks]);

View File

@ -18,22 +18,37 @@
package org.apache.hadoop.hdfs.server.blockmanagement; package org.apache.hadoop.hdfs.server.blockmanagement;
import java.util.Collection;
import java.util.Iterator; import java.util.Iterator;
import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.StripedFileTestUtil;
import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertFalse;
import static org.junit.Assert.fail; import static org.junit.Assert.fail;
/**
* Test {@link LowRedundancyBlocks}.
*/
@RunWith(Parameterized.class)
public class TestLowRedundancyBlockQueues { public class TestLowRedundancyBlockQueues {
private final ErasureCodingPolicy ecPolicy = private final ErasureCodingPolicy ecPolicy;
StripedFileTestUtil.getDefaultECPolicy();
public TestLowRedundancyBlockQueues(ErasureCodingPolicy policy) {
ecPolicy = policy;
}
@Parameterized.Parameters(name = "{index}: {0}")
public static Collection<Object[]> policies() {
return StripedFileTestUtil.getECPolicies();
}
private BlockInfo genBlockInfo(long id) { private BlockInfo genBlockInfo(long id) {
return new BlockInfoContiguous(new Block(id), (short) 3); return new BlockInfoContiguous(new Block(id), (short) 3);