From 9b61ecfcbeeb1d3c320f775b43c68d3e39a271a3 Mon Sep 17 00:00:00 2001 From: huzheng Date: Tue, 4 Jun 2019 10:54:46 +0800 Subject: [PATCH] HDFS-14535. The default 8KB buffer in requestFileDescriptors#BufferedOutputStream is causing lots of heap allocation in HBase when using short-circut read (cherry picked from commit ea3b0a184405c3feca024a560807ea215b6858b9) --- .../org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java index 8e592f4a54e..88b168621db 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java @@ -86,6 +86,7 @@ import org.slf4j.LoggerFactory; @InterfaceAudience.Private public class BlockReaderFactory implements ShortCircuitReplicaCreator { static final Logger LOG = LoggerFactory.getLogger(BlockReaderFactory.class); + private static final int SMALL_BUFFER_SIZE = 512; public static class FailureInjector { public void injectRequestFileDescriptorsFailure() throws IOException { @@ -582,7 +583,7 @@ public class BlockReaderFactory implements ShortCircuitReplicaCreator { Slot slot) throws IOException { ShortCircuitCache cache = clientContext.getShortCircuitCache(); final DataOutputStream out = - new DataOutputStream(new BufferedOutputStream(peer.getOutputStream())); + new DataOutputStream(new BufferedOutputStream(peer.getOutputStream(), SMALL_BUFFER_SIZE)); SlotId slotId = slot == null ? null : slot.getSlotId(); new Sender(out).requestShortCircuitFds(block, token, slotId, 1, failureInjector.getSupportsReceiptVerification());