From a3e16f414c86c63a9bfcae7d41256911f1b2151a Mon Sep 17 00:00:00 2001 From: Jing Zhao Date: Fri, 20 Mar 2015 10:50:03 -0700 Subject: [PATCH] HDFS-7829. Code clean up for LocatedBlock. Contributed by Takanobu Asanuma. (cherry picked from commit a6a5aae472d015d2ea5cd746719485dff93873a8) --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../hadoop/hdfs/protocol/LocatedBlock.java | 20 ++++++------------- .../hadoop/hdfs/protocol/LocatedBlocks.java | 2 +- .../hdfs/server/namenode/FSNamesystem.java | 2 +- .../server/protocol/BlockRecoveryCommand.java | 4 ++-- .../hadoop/hdfs/TestDFSClientRetries.java | 5 ++--- .../org/apache/hadoop/hdfs/TestDFSUtil.java | 8 ++++++-- .../hadoop/hdfs/protocolPB/TestPBHelper.java | 3 ++- 8 files changed, 22 insertions(+), 24 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 402504d57a5..cd6aff55130 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -13,6 +13,8 @@ Release 2.8.0 - UNRELEASED HDFS-7835. make initial sleeptime in locateFollowingBlock configurable for DFSClient. (Zhihai Xu via Yongjun Zhang) + HDFS-7829. Code clean up for LocatedBlock. (Takanobu Asanuma via jing9) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java index 0d521918484..e729869878f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java @@ -44,9 +44,9 @@ public class LocatedBlock { private long offset; // offset of the first byte of the block in the file private final DatanodeInfoWithStorage[] locs; /** Cached storage ID for each replica */ - private String[] storageIDs; + private final String[] storageIDs; /** Cached storage type for each replica, if reported. */ - private StorageType[] storageTypes; + private final StorageType[] storageTypes; // corrupt flag is true if all of the replicas of a block are corrupt. // else false. If block has few corrupt replicas, they are filtered and // their locations are not part of this object @@ -62,16 +62,8 @@ public class LocatedBlock { new DatanodeInfoWithStorage[0]; public LocatedBlock(ExtendedBlock b, DatanodeInfo[] locs) { - this(b, locs, -1, false); // startOffset is unknown - } - - public LocatedBlock(ExtendedBlock b, DatanodeInfo[] locs, long startOffset, - boolean corrupt) { - this(b, locs, null, null, startOffset, corrupt, EMPTY_LOCS); - } - - public LocatedBlock(ExtendedBlock b, DatanodeStorageInfo[] storages) { - this(b, storages, -1, false); // startOffset is unknown + // By default, startOffset is unknown(-1) and corrupt is false. + this(b, locs, null, null, -1, false, EMPTY_LOCS); } public LocatedBlock(ExtendedBlock b, DatanodeInfo[] locs, @@ -170,11 +162,11 @@ public long getBlockSize() { return b.getNumBytes(); } - void setStartOffset(long value) { + public void setStartOffset(long value) { this.offset = value; } - void setCorrupt(boolean corrupt) { + public void setCorrupt(boolean corrupt) { this.corrupt = corrupt; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java index fc739cf711e..e35a4310744 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java @@ -119,7 +119,7 @@ public FileEncryptionInfo getFileEncryptionInfo() { public int findBlock(long offset) { // create fake block of size 0 as a key LocatedBlock key = new LocatedBlock( - new ExtendedBlock(), new DatanodeInfo[0], 0L, false); + new ExtendedBlock(), new DatanodeInfo[0]); key.setStartOffset(offset); key.getBlock().setNumBytes(1); Comparator comp = diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index ad17e8cf1a9..7dbc2dcfed1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -3292,7 +3292,7 @@ LocatedBlock getAdditionalDatanode(String src, long fileId, final DatanodeStorageInfo[] targets = blockManager.chooseTarget4AdditionalDatanode( src, numAdditionalNodes, clientnode, chosen, excludes, preferredblocksize, storagePolicyID); - final LocatedBlock lb = new LocatedBlock(blk, targets); + final LocatedBlock lb = new LocatedBlock(blk, targets, -1, false); blockManager.setBlockToken(lb, AccessMode.COPY); return lb; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockRecoveryCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockRecoveryCommand.java index ced3296f486..3adc85c5a21 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockRecoveryCommand.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockRecoveryCommand.java @@ -61,7 +61,7 @@ public static class RecoveringBlock extends LocatedBlock { * Create RecoveringBlock. */ public RecoveringBlock(ExtendedBlock b, DatanodeInfo[] locs, long newGS) { - super(b, locs, -1, false); // startOffset is unknown + super(b, locs); // startOffset is unknown this.newGenerationStamp = newGS; this.recoveryBlock = null; } @@ -71,7 +71,7 @@ public RecoveringBlock(ExtendedBlock b, DatanodeInfo[] locs, long newGS) { */ public RecoveringBlock(ExtendedBlock b, DatanodeInfo[] locs, Block recoveryBlock) { - super(b, locs, -1, false); // startOffset is unknown + super(b, locs); // startOffset is unknown this.newGenerationStamp = recoveryBlock.getGenerationStamp(); this.recoveryBlock = recoveryBlock; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java index 45f21dfd7fd..18f7c77e185 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java @@ -489,9 +489,8 @@ private LocatedBlocks makeBadBlockList(LocatedBlocks goodBlockList) { goodLocatedBlock.getBlock(), new DatanodeInfo[] { DFSTestUtil.getDatanodeInfo("1.2.3.4", "bogus", 1234) - }, - goodLocatedBlock.getStartOffset(), - false); + }); + badLocatedBlock.setStartOffset(goodLocatedBlock.getStartOffset()); List badBlocks = new ArrayList(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java index 046265f5520..7de121f4193 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java @@ -99,11 +99,15 @@ public void testLocatedBlocks2Locations() { // ok ExtendedBlock b1 = new ExtendedBlock("bpid", 1, 1, 1); - LocatedBlock l1 = new LocatedBlock(b1, ds, 0, false); + LocatedBlock l1 = new LocatedBlock(b1, ds); + l1.setStartOffset(0); + l1.setCorrupt(false); // corrupt ExtendedBlock b2 = new ExtendedBlock("bpid", 2, 1, 1); - LocatedBlock l2 = new LocatedBlock(b2, ds, 0, true); + LocatedBlock l2 = new LocatedBlock(b2, ds); + l2.setStartOffset(0); + l2.setCorrupt(true); List ls = Arrays.asList(l1, l2); LocatedBlocks lbs = new LocatedBlocks(10, false, ls, l2, true, null); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java index 0236288a420..c7233bd5da9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java @@ -478,10 +478,11 @@ private LocatedBlock createLocatedBlockNoStorageMedia() { AdminStates.NORMAL) }; LocatedBlock lb = new LocatedBlock( - new ExtendedBlock("bp12", 12345, 10, 53), dnInfos, 5, false); + new ExtendedBlock("bp12", 12345, 10, 53), dnInfos); lb.setBlockToken(new Token( "identifier".getBytes(), "password".getBytes(), new Text("kind"), new Text("service"))); + lb.setStartOffset(5); return lb; }