Apache/hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-14535. The default 8KB buffer in requestFileDescriptors#BufferedOutputStream is causing lots of heap allocation in HBase when using short-circut read

Browse Source 
(cherry picked from commit ea3b0a184405c3feca024a560807ea215b6858b9)
(cherry picked from commit 9b61ecfcbeeb1d3c320f775b43c68d3e39a271a3)
(cherry picked from commit b082628e5a39fc1bc9d6094beb9c253cc096c701)
(cherry picked from commit 2b9c4fccf4a8930bfff99ab4928b9b3b4980d3b2)
This commit is contained in:
huzheng 2019-06-04 10:54:46 +08:00 committed by Wei-Chiu Chuang
parent 9ab3cbf4d3
commit 8b90c69357
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java
View File

@ -87,6 +87,7 @@
@InterfaceAudience.Private @InterfaceAudience.Private
public class BlockReaderFactory implements ShortCircuitReplicaCreator { public class BlockReaderFactory implements ShortCircuitReplicaCreator {
static final Logger LOG = LoggerFactory.getLogger(BlockReaderFactory.class); static final Logger LOG = LoggerFactory.getLogger(BlockReaderFactory.class);
private static final int SMALL_BUFFER_SIZE = 512;
public static class FailureInjector { public static class FailureInjector {
public void injectRequestFileDescriptorsFailure() throws IOException { public void injectRequestFileDescriptorsFailure() throws IOException {
@ -590,7 +591,7 @@ private ShortCircuitReplicaInfo requestFileDescriptors(DomainPeer peer,
Slot slot) throws IOException { Slot slot) throws IOException {
ShortCircuitCache cache = clientContext.getShortCircuitCache(); ShortCircuitCache cache = clientContext.getShortCircuitCache();
final DataOutputStream out = final DataOutputStream out =
new DataOutputStream(new BufferedOutputStream(peer.getOutputStream())); new DataOutputStream(new BufferedOutputStream(peer.getOutputStream(), SMALL_BUFFER_SIZE));
SlotId slotId = slot == null ? null : slot.getSlotId(); SlotId slotId = slot == null ? null : slot.getSlotId();
new Sender(out).requestShortCircuitFds(block, token, slotId, 1, new Sender(out).requestShortCircuitFds(block, token, slotId, 1,
failureInjector.getSupportsReceiptVerification()); failureInjector.getSupportsReceiptVerification());