diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java index f35f6a58a4e..d64bdd95f03 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java @@ -189,7 +189,8 @@ public class PBHelperClient { } public static ByteString getByteString(byte[] bytes) { - return ByteString.copyFrom(bytes); + // return singleton to reduce object allocation + return (bytes.length == 0) ? ByteString.EMPTY : ByteString.copyFrom(bytes); } public static ShmId convert(ShortCircuitShmIdProto shmId) { @@ -216,8 +217,8 @@ public class PBHelperClient { public static TokenProto convert(Token tok) { return TokenProto.newBuilder(). - setIdentifier(ByteString.copyFrom(tok.getIdentifier())). - setPassword(ByteString.copyFrom(tok.getPassword())). + setIdentifier(getByteString(tok.getIdentifier())). + setPassword(getByteString(tok.getPassword())). setKind(tok.getKind().toString()). setService(tok.getService().toString()).build(); } @@ -446,16 +447,16 @@ public class PBHelperClient { builder.setSuite(convert(option.getCipherSuite())); } if (option.getInKey() != null) { - builder.setInKey(ByteString.copyFrom(option.getInKey())); + builder.setInKey(getByteString(option.getInKey())); } if (option.getInIv() != null) { - builder.setInIv(ByteString.copyFrom(option.getInIv())); + builder.setInIv(getByteString(option.getInIv())); } if (option.getOutKey() != null) { - builder.setOutKey(ByteString.copyFrom(option.getOutKey())); + builder.setOutKey(getByteString(option.getOutKey())); } if (option.getOutIv() != null) { - builder.setOutIv(ByteString.copyFrom(option.getOutIv())); + builder.setOutIv(getByteString(option.getOutIv())); } return builder.build(); } @@ -1676,8 +1677,8 @@ public class PBHelperClient { DataEncryptionKeyProto.Builder b = DataEncryptionKeyProto.newBuilder() .setKeyId(bet.keyId) .setBlockPoolId(bet.blockPoolId) - .setNonce(ByteString.copyFrom(bet.nonce)) - .setEncryptionKey(ByteString.copyFrom(bet.encryptionKey)) + .setNonce(getByteString(bet.nonce)) + .setEncryptionKey(getByteString(bet.encryptionKey)) .setExpiryDate(bet.expiryDate); if (bet.encryptionAlgorithm != null) { b.setEncryptionAlgorithm(bet.encryptionAlgorithm); @@ -1754,10 +1755,10 @@ public class PBHelperClient { setGroup(fs.getGroup()). setFileId(fs.getFileId()). setChildrenNum(fs.getChildrenNum()). - setPath(ByteString.copyFrom(fs.getLocalNameInBytes())). + setPath(getByteString(fs.getLocalNameInBytes())). setStoragePolicy(fs.getStoragePolicy()); if (fs.isSymlink()) { - builder.setSymlink(ByteString.copyFrom(fs.getSymlinkInBytes())); + builder.setSymlink(getByteString(fs.getSymlinkInBytes())); } if (fs.getFileEncryptionInfo() != null) { builder.setFileEncryptionInfo(convert(fs.getFileEncryptionInfo())); @@ -1780,7 +1781,7 @@ public class PBHelperClient { int snapshotNumber = status.getSnapshotNumber(); int snapshotQuota = status.getSnapshotQuota(); byte[] parentFullPath = status.getParentFullPath(); - ByteString parentFullPathBytes = ByteString.copyFrom( + ByteString parentFullPathBytes = getByteString( parentFullPath == null ? DFSUtilClient.EMPTY_BYTES : parentFullPath); HdfsFileStatusProto fs = convert(status.getDirStatus()); SnapshottableDirectoryStatusProto.Builder builder = @@ -1995,7 +1996,7 @@ public class PBHelperClient { if (entry == null) { return null; } - ByteString sourcePath = ByteString.copyFrom(entry.getSourcePath() == null ? + ByteString sourcePath = getByteString(entry.getSourcePath() == null ? DFSUtilClient.EMPTY_BYTES : entry.getSourcePath()); String modification = entry.getType().getLabel(); SnapshotDiffReportEntryProto.Builder builder = SnapshotDiffReportEntryProto @@ -2003,7 +2004,7 @@ public class PBHelperClient { .setModificationLabel(modification); if (entry.getType() == DiffType.RENAME) { ByteString targetPath = - ByteString.copyFrom(entry.getTargetPath() == null ? + getByteString(entry.getTargetPath() == null ? DFSUtilClient.EMPTY_BYTES : entry.getTargetPath()); builder.setTargetPath(targetPath); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index aee4defa67e..994f28e1529 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -859,6 +859,9 @@ Release 2.8.0 - UNRELEASED HDFS-8894. Set SO_KEEPALIVE on DN server sockets. (Kanaka Kumar Avvaru via wang) + HDFS-9557. Reduce object allocation in PB conversion + (Daryn Sharp via cnauroth) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java index a3bf0debd32..dab62e3188e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java @@ -209,7 +209,7 @@ public class PBHelper { public static BlockKeyProto convert(BlockKey key) { byte[] encodedKey = key.getEncodedKey(); - ByteString keyBytes = ByteString.copyFrom(encodedKey == null ? + ByteString keyBytes = PBHelperClient.getByteString(encodedKey == null ? DFSUtilClient.EMPTY_BYTES : encodedKey); return BlockKeyProto.newBuilder().setKeyId(key.getKeyId()) .setKeyBytes(keyBytes).setExpiryDate(key.getExpiryDate()).build(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java index 2bfba98cf12..aa93328b77a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.protocolPB; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @@ -89,6 +90,7 @@ import org.junit.Test; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; +import com.google.protobuf.ByteString; /** * Tests for {@link PBHelper} @@ -100,6 +102,11 @@ public class TestPBHelper { */ private static final double DELTA = 0.000001; + @Test + public void testGetByteString() { + assertSame(ByteString.EMPTY, PBHelperClient.getByteString(new byte[0])); + } + @Test public void testConvertNamenodeRole() { assertEquals(NamenodeRoleProto.BACKUP,