Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-9557. Reduce object allocation in PB conversion. Contributed by Daryn Sharp. 

(cherry picked from commit c470c8953d)
This commit is contained in:
cnauroth 2015-12-16 10:45:32 -08:00
parent 1a2ef845b5
commit e8e3737ce8
4 changed files with 26 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
View File

@ -189,7 +189,8 @@ public class PBHelperClient {
}
public static ByteString getByteString(byte[] bytes) {
return ByteString.copyFrom(bytes);
// return singleton to reduce object allocation
return (bytes.length == 0) ? ByteString.EMPTY : ByteString.copyFrom(bytes);
}
public static ShmId convert(ShortCircuitShmIdProto shmId) {
@ -216,8 +217,8 @@ public class PBHelperClient {
public static TokenProto convert(Token<?> tok) {
return TokenProto.newBuilder().
setIdentifier(ByteString.copyFrom(tok.getIdentifier())).
setPassword(ByteString.copyFrom(tok.getPassword())).
setIdentifier(getByteString(tok.getIdentifier())).
setPassword(getByteString(tok.getPassword())).
setKind(tok.getKind().toString()).
setService(tok.getService().toString()).build();
}
@ -446,16 +447,16 @@ public class PBHelperClient {
builder.setSuite(convert(option.getCipherSuite()));
}
if (option.getInKey() != null) {
builder.setInKey(ByteString.copyFrom(option.getInKey()));
builder.setInKey(getByteString(option.getInKey()));
}
if (option.getInIv() != null) {
builder.setInIv(ByteString.copyFrom(option.getInIv()));
builder.setInIv(getByteString(option.getInIv()));
}
if (option.getOutKey() != null) {
builder.setOutKey(ByteString.copyFrom(option.getOutKey()));
builder.setOutKey(getByteString(option.getOutKey()));
}
if (option.getOutIv() != null) {
builder.setOutIv(ByteString.copyFrom(option.getOutIv()));
builder.setOutIv(getByteString(option.getOutIv()));
}
return builder.build();
}
@ -1676,8 +1677,8 @@ public class PBHelperClient {
DataEncryptionKeyProto.Builder b = DataEncryptionKeyProto.newBuilder()
.setKeyId(bet.keyId)
.setBlockPoolId(bet.blockPoolId)
.setNonce(ByteString.copyFrom(bet.nonce))
.setEncryptionKey(ByteString.copyFrom(bet.encryptionKey))
.setNonce(getByteString(bet.nonce))
.setEncryptionKey(getByteString(bet.encryptionKey))
.setExpiryDate(bet.expiryDate);
if (bet.encryptionAlgorithm != null) {
b.setEncryptionAlgorithm(bet.encryptionAlgorithm);
@ -1754,10 +1755,10 @@ public class PBHelperClient {
setGroup(fs.getGroup()).
setFileId(fs.getFileId()).
setChildrenNum(fs.getChildrenNum()).
setPath(ByteString.copyFrom(fs.getLocalNameInBytes())).
setPath(getByteString(fs.getLocalNameInBytes())).
setStoragePolicy(fs.getStoragePolicy());
if (fs.isSymlink()) {
builder.setSymlink(ByteString.copyFrom(fs.getSymlinkInBytes()));
builder.setSymlink(getByteString(fs.getSymlinkInBytes()));
}
if (fs.getFileEncryptionInfo() != null) {
builder.setFileEncryptionInfo(convert(fs.getFileEncryptionInfo()));
@ -1780,7 +1781,7 @@ public class PBHelperClient {
int snapshotNumber = status.getSnapshotNumber();
int snapshotQuota = status.getSnapshotQuota();
byte[] parentFullPath = status.getParentFullPath();
ByteString parentFullPathBytes = ByteString.copyFrom(
ByteString parentFullPathBytes = getByteString(
parentFullPath == null ? DFSUtilClient.EMPTY_BYTES : parentFullPath);
HdfsFileStatusProto fs = convert(status.getDirStatus());
SnapshottableDirectoryStatusProto.Builder builder =
@ -1995,7 +1996,7 @@ public class PBHelperClient {
if (entry == null) {
return null;
}
ByteString sourcePath = ByteString.copyFrom(entry.getSourcePath() == null ?
ByteString sourcePath = getByteString(entry.getSourcePath() == null ?
DFSUtilClient.EMPTY_BYTES : entry.getSourcePath());
String modification = entry.getType().getLabel();
SnapshotDiffReportEntryProto.Builder builder = SnapshotDiffReportEntryProto
@ -2003,7 +2004,7 @@ public class PBHelperClient {
.setModificationLabel(modification);
if (entry.getType() == DiffType.RENAME) {
ByteString targetPath =
ByteString.copyFrom(entry.getTargetPath() == null ?
getByteString(entry.getTargetPath() == null ?
DFSUtilClient.EMPTY_BYTES : entry.getTargetPath());
builder.setTargetPath(targetPath);
}

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -898,6 +898,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8894. Set SO_KEEPALIVE on DN server sockets.
(Kanaka Kumar Avvaru via wang)
HDFS-9557. Reduce object allocation in PB conversion
(Daryn Sharp via cnauroth)
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
View File

@ -209,7 +209,7 @@ public class PBHelper {
public static BlockKeyProto convert(BlockKey key) {
byte[] encodedKey = key.getEncodedKey();
ByteString keyBytes = ByteString.copyFrom(encodedKey == null ?
ByteString keyBytes = PBHelperClient.getByteString(encodedKey == null ?
DFSUtilClient.EMPTY_BYTES : encodedKey);
return BlockKeyProto.newBuilder().setKeyId(key.getKeyId())
.setKeyBytes(keyBytes).setExpiryDate(key.getExpiryDate()).build();

7
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
View File

@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.protocolPB;
import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
@ -89,6 +90,7 @@ import org.junit.Test;
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.protobuf.ByteString;
/**
* Tests for {@link PBHelper}
@ -100,6 +102,11 @@ public class TestPBHelper {
*/
private static final double DELTA = 0.000001;
@Test
public void testGetByteString() {
assertSame(ByteString.EMPTY, PBHelperClient.getByteString(new byte[0]));
}
@Test
public void testConvertNamenodeRole() {
assertEquals(NamenodeRoleProto.BACKUP,