HDFS-12392. Writing striped file failed due to different cell size. Contributed by Sammi Chen
This commit is contained in:
parent
d4035d42f0
commit
d7f27043ce
|
@ -215,6 +215,7 @@ public class DFSStripedOutputStream extends DFSOutputStream
|
||||||
buffers = new ByteBuffer[numAllBlocks];
|
buffers = new ByteBuffer[numAllBlocks];
|
||||||
for (int i = 0; i < buffers.length; i++) {
|
for (int i = 0; i < buffers.length; i++) {
|
||||||
buffers[i] = BUFFER_POOL.getBuffer(useDirectBuffer(), cellSize);
|
buffers[i] = BUFFER_POOL.getBuffer(useDirectBuffer(), cellSize);
|
||||||
|
buffers[i].limit(cellSize);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -237,6 +238,7 @@ public class DFSStripedOutputStream extends DFSOutputStream
|
||||||
private void clear() {
|
private void clear() {
|
||||||
for (int i = 0; i< numAllBlocks; i++) {
|
for (int i = 0; i< numAllBlocks; i++) {
|
||||||
buffers[i].clear();
|
buffers[i].clear();
|
||||||
|
buffers[i].limit(cellSize);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -35,6 +35,7 @@ import org.apache.hadoop.hdfs.client.HdfsAdmin;
|
||||||
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
|
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
|
||||||
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
|
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
|
||||||
import org.apache.hadoop.io.erasurecode.ECSchema;
|
import org.apache.hadoop.io.erasurecode.ECSchema;
|
||||||
|
import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
|
||||||
import org.apache.hadoop.security.AccessControlException;
|
import org.apache.hadoop.security.AccessControlException;
|
||||||
import org.apache.hadoop.security.UserGroupInformation;
|
import org.apache.hadoop.security.UserGroupInformation;
|
||||||
import org.apache.hadoop.test.GenericTestUtils;
|
import org.apache.hadoop.test.GenericTestUtils;
|
||||||
|
@ -60,7 +61,7 @@ public class TestErasureCodingPolicies {
|
||||||
private Configuration conf;
|
private Configuration conf;
|
||||||
private MiniDFSCluster cluster;
|
private MiniDFSCluster cluster;
|
||||||
private DistributedFileSystem fs;
|
private DistributedFileSystem fs;
|
||||||
private static final int BLOCK_SIZE = 1024;
|
private static final int BLOCK_SIZE = 16 * 1024;
|
||||||
private ErasureCodingPolicy ecPolicy;
|
private ErasureCodingPolicy ecPolicy;
|
||||||
private FSNamesystem namesystem;
|
private FSNamesystem namesystem;
|
||||||
|
|
||||||
|
@ -78,7 +79,8 @@ public class TestErasureCodingPolicies {
|
||||||
conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
|
conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
|
||||||
DFSTestUtil.enableAllECPolicies(conf);
|
DFSTestUtil.enableAllECPolicies(conf);
|
||||||
cluster = new MiniDFSCluster.Builder(conf).
|
cluster = new MiniDFSCluster.Builder(conf).
|
||||||
numDataNodes(1).build();
|
numDataNodes(ecPolicy.getNumDataUnits() + ecPolicy.getNumParityUnits()).
|
||||||
|
build();
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
fs = cluster.getFileSystem();
|
fs = cluster.getFileSystem();
|
||||||
namesystem = cluster.getNamesystem();
|
namesystem = cluster.getNamesystem();
|
||||||
|
@ -836,4 +838,48 @@ public class TestErasureCodingPolicies {
|
||||||
ecPolicy, fs.getErasureCodingPolicy(subReplicaFile));
|
ecPolicy, fs.getErasureCodingPolicy(subReplicaFile));
|
||||||
fs.delete(subReplicaFile, false);
|
fs.delete(subReplicaFile, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testDifferentErasureCodingPolicyCellSize() throws Exception {
|
||||||
|
// add policy with cell size 8K
|
||||||
|
ErasureCodingPolicy newPolicy1 =
|
||||||
|
new ErasureCodingPolicy(ErasureCodeConstants.RS_3_2_SCHEMA, 8 * 1024);
|
||||||
|
ErasureCodingPolicy[] policyArray =
|
||||||
|
new ErasureCodingPolicy[] {newPolicy1};
|
||||||
|
AddECPolicyResponse[] responses = fs.addErasureCodingPolicies(policyArray);
|
||||||
|
assertEquals(1, responses.length);
|
||||||
|
assertTrue(responses[0].isSucceed());
|
||||||
|
newPolicy1 = responses[0].getPolicy();
|
||||||
|
|
||||||
|
// add policy with cell size 4K
|
||||||
|
ErasureCodingPolicy newPolicy2 =
|
||||||
|
new ErasureCodingPolicy(ErasureCodeConstants.RS_3_2_SCHEMA, 4 * 1024);
|
||||||
|
policyArray = new ErasureCodingPolicy[] {newPolicy2};
|
||||||
|
responses = fs.addErasureCodingPolicies(policyArray);
|
||||||
|
assertEquals(1, responses.length);
|
||||||
|
assertTrue(responses[0].isSucceed());
|
||||||
|
newPolicy2 = responses[0].getPolicy();
|
||||||
|
|
||||||
|
// enable policies
|
||||||
|
fs.enableErasureCodingPolicy(newPolicy1.getName());
|
||||||
|
fs.enableErasureCodingPolicy(newPolicy2.getName());
|
||||||
|
|
||||||
|
final Path stripedDir1 = new Path("/striped1");
|
||||||
|
final Path stripedDir2 = new Path("/striped2");
|
||||||
|
final Path file1 = new Path(stripedDir1, "file");
|
||||||
|
final Path file2 = new Path(stripedDir2, "file");
|
||||||
|
|
||||||
|
fs.mkdirs(stripedDir1);
|
||||||
|
fs.setErasureCodingPolicy(stripedDir1, newPolicy1.getName());
|
||||||
|
fs.mkdirs(stripedDir2);
|
||||||
|
fs.setErasureCodingPolicy(stripedDir2, newPolicy2.getName());
|
||||||
|
|
||||||
|
final int fileLength = BLOCK_SIZE * newPolicy1.getNumDataUnits();
|
||||||
|
final byte[] bytes = StripedFileTestUtil.generateBytes(fileLength);
|
||||||
|
DFSTestUtil.writeFile(fs, file1, bytes);
|
||||||
|
DFSTestUtil.writeFile(fs, file2, bytes);
|
||||||
|
|
||||||
|
fs.delete(stripedDir1, true);
|
||||||
|
fs.delete(stripedDir2, true);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue