HDFS-8223. Should calculate checksum for parity blocks in DFSStripedOutputStream. Contributed by Yi Liu.
This commit is contained in:
parent
eb612b0b70
commit
3f2c6938f1
|
@ -196,6 +196,10 @@ abstract public class FSOutputSummer extends OutputStream {
|
|||
return sum.getChecksumSize();
|
||||
}
|
||||
|
||||
protected DataChecksum getDataChecksum() {
|
||||
return sum;
|
||||
}
|
||||
|
||||
protected TraceScope createWriteTraceScope() {
|
||||
return NullScope.INSTANCE;
|
||||
}
|
||||
|
|
|
@ -125,3 +125,6 @@
|
|||
|
||||
HDFS-8233. Fix DFSStripedOutputStream#getCurrentBlockGroupBytes when the last
|
||||
stripe is at the block group boundary. (jing9)
|
||||
|
||||
HDFS-8223. Should calculate checksum for parity blocks in DFSStripedOutputStream.
|
||||
(Yi Liu via jing9)
|
||||
|
|
|
@ -62,6 +62,8 @@ public class DFSStripedOutputStream extends DFSOutputStream {
|
|||
*/
|
||||
private final ECInfo ecInfo;
|
||||
private final int cellSize;
|
||||
// checksum buffer, we only need to calculate checksum for parity blocks
|
||||
private byte[] checksumBuf;
|
||||
private ByteBuffer[] cellBuffers;
|
||||
|
||||
private final short numAllBlocks;
|
||||
|
@ -99,6 +101,7 @@ public class DFSStripedOutputStream extends DFSOutputStream {
|
|||
|
||||
checkConfiguration();
|
||||
|
||||
checksumBuf = new byte[getChecksumSize() * (cellSize / bytesPerChecksum)];
|
||||
cellBuffers = new ByteBuffer[numAllBlocks];
|
||||
List<BlockingQueue<LocatedBlock>> stripeBlocks = new ArrayList<>();
|
||||
|
||||
|
@ -179,6 +182,10 @@ public class DFSStripedOutputStream extends DFSOutputStream {
|
|||
private List<DFSPacket> generatePackets(ByteBuffer byteBuffer)
|
||||
throws IOException{
|
||||
List<DFSPacket> packets = new ArrayList<>();
|
||||
assert byteBuffer.hasArray();
|
||||
getDataChecksum().calculateChunkedSums(byteBuffer.array(), 0,
|
||||
byteBuffer.remaining(), checksumBuf, 0);
|
||||
int ckOff = 0;
|
||||
while (byteBuffer.remaining() > 0) {
|
||||
DFSPacket p = createPacket(packetSize, chunksPerPacket,
|
||||
streamer.getBytesCurBlock(),
|
||||
|
@ -186,6 +193,9 @@ public class DFSStripedOutputStream extends DFSOutputStream {
|
|||
int maxBytesToPacket = p.getMaxChunks() * bytesPerChecksum;
|
||||
int toWrite = byteBuffer.remaining() > maxBytesToPacket ?
|
||||
maxBytesToPacket: byteBuffer.remaining();
|
||||
int ckLen = ((toWrite - 1) / bytesPerChecksum + 1) * getChecksumSize();
|
||||
p.writeChecksum(checksumBuf, ckOff, ckLen);
|
||||
ckOff += ckLen;
|
||||
p.writeData(byteBuffer, toWrite);
|
||||
streamer.incBytesCurBlock(toWrite);
|
||||
packets.add(p);
|
||||
|
|
Loading…
Reference in New Issue