From c09dc258a8f64fab852bf6f26187163480dbee3c Mon Sep 17 00:00:00 2001 From: Zhe Zhang Date: Wed, 23 Sep 2015 13:34:09 -0700 Subject: [PATCH] HDFS-8882. Erasure Coding: Use datablocks, parityblocks and cell size from ErasureCodingPolicy. Contributed by Vinayakumar B. Change-Id: Ic56da0b426f47c63dac440aef6f5fc8554f6cf13 --- .../hadoop/hdfs/protocol/HdfsConstants.java | 11 ----- .../src/main/proto/hdfs.proto | 1 + .../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 5 ++- .../hadoop/hdfs/StripedDataStreamer.java | 34 ++++++++-------- .../hadoop/hdfs/protocolPB/PBHelper.java | 3 +- .../hdfs/server/balancer/Dispatcher.java | 13 +++--- .../blockmanagement/BlockInfoStriped.java | 10 +++-- .../server/blockmanagement/BlockManager.java | 10 ++--- .../hadoop/hdfs/server/mover/Mover.java | 3 +- .../server/namenode/FSDirWriteFileOp.java | 11 +++-- .../hdfs/server/namenode/INodeFile.java | 11 +++-- .../server/protocol/BlocksWithLocations.java | 9 ++++- .../org/apache/hadoop/hdfs/DFSTestUtil.java | 4 +- .../hadoop/hdfs/StripedFileTestUtil.java | 40 +++++++++---------- .../hdfs/TestDFSStripedInputStream.java | 7 ++-- .../hdfs/TestDFSStripedOutputStream.java | 7 ++-- ...TestDFSStripedOutputStreamWithFailure.java | 8 ++-- .../hdfs/TestReadStripedFileWithDecoding.java | 6 +-- .../TestReadStripedFileWithMissingBlocks.java | 6 +-- .../hadoop/hdfs/TestRecoverStripedFile.java | 6 +-- .../hdfs/TestSafeModeWithStripedFile.java | 6 +-- .../hadoop/hdfs/TestWriteReadStripedFile.java | 4 +- .../hdfs/TestWriteStripedFileWithFailure.java | 4 +- .../hadoop/hdfs/protocolPB/TestPBHelper.java | 4 +- .../hdfs/server/balancer/TestBalancer.java | 7 ++-- .../blockmanagement/TestBlockInfoStriped.java | 4 +- .../TestBlockTokenWithDFSStriped.java | 8 ++-- .../TestSequentialBlockGroupId.java | 8 ++-- .../hadoop/hdfs/server/mover/TestMover.java | 7 ++-- .../TestAddOverReplicatedStripedBlocks.java | 9 +++-- .../server/namenode/TestAddStripedBlocks.java | 13 +++--- .../server/namenode/TestFSEditLogLoader.java | 9 +++-- .../hdfs/server/namenode/TestFSImage.java | 9 +++-- .../namenode/TestQuotaWithStripedBlocks.java | 5 ++- .../namenode/TestRecoverStripedBlocks.java | 8 ++-- .../server/namenode/TestStripedINodeFile.java | 5 ++- ...stOfflineImageViewerWithStripedBlocks.java | 8 ++-- .../hdfs/util/TestStripedBlockUtil.java | 10 ++--- 38 files changed, 177 insertions(+), 156 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java index a8bb7407bc0..0453d1d3530 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java @@ -86,17 +86,6 @@ public final class HdfsConstants { public static final String CLIENT_NAMENODE_PROTOCOL_NAME = "org.apache.hadoop.hdfs.protocol.ClientProtocol"; - /* - * These values correspond to the values used by the system default erasure - * coding policy. - * TODO: get these values from ec policy of the associated INodeFile - */ - - public static final byte NUM_DATA_BLOCKS = 6; - public static final byte NUM_PARITY_BLOCKS = 3; - // The chunk size for striped block which is used by erasure coding - public static final int BLOCK_STRIPED_CELL_SIZE = 64 * 1024; - // Timeouts for communicating with DataNode for streaming writes/reads public static final int READ_TIMEOUT = 60 * 1000; public static final int READ_TIMEOUT_EXTENSION = 5 * 1000; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto index 77c64b9f8e7..b28ab425c8d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto @@ -536,6 +536,7 @@ message BlockWithLocationsProto { optional bytes indices = 5; optional uint32 dataBlockNum = 6; + optional uint32 cellSize = 7; } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index b79ce64ac89..d62dbacd4ed 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -446,4 +446,7 @@ getAllErasureCodingPolicies. (Rakesh R via zhz) HDFS-8920. Erasure Coding: when recovering lost blocks, logs can be too - verbose and hurt performance. (Rui Li via Kai Zheng) \ No newline at end of file + verbose and hurt performance. (Rui Li via Kai Zheng) + + HDFS-8882. Erasure Coding: Use datablocks, parityblocks and cell size from + ErasureCodingPolicy (Vinayakumar B via zhz) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StripedDataStreamer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StripedDataStreamer.java index a2b6a676d4e..2f83f7c6418 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StripedDataStreamer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StripedDataStreamer.java @@ -18,25 +18,20 @@ package org.apache.hadoop.hdfs; -import static org.apache.hadoop.hdfs.protocol.HdfsConstants.BLOCK_STRIPED_CELL_SIZE; -import static org.apache.hadoop.hdfs.protocol.HdfsConstants.NUM_DATA_BLOCKS; -import static org.apache.hadoop.hdfs.protocol.HdfsConstants.NUM_PARITY_BLOCKS; - import java.io.IOException; import java.io.InterruptedIOException; import java.util.concurrent.atomic.AtomicReference; - import org.apache.hadoop.hdfs.DFSStripedOutputStream.Coordinator; import org.apache.hadoop.hdfs.DFSStripedOutputStream.MultipleBlockingQueue; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock; import org.apache.hadoop.hdfs.server.datanode.CachingStrategy; import org.apache.hadoop.hdfs.util.ByteArrayManager; import org.apache.hadoop.hdfs.util.StripedBlockUtil; +import org.apache.hadoop.io.erasurecode.ECSchema; import org.apache.hadoop.util.DataChecksum; import org.apache.hadoop.util.Progressable; @@ -109,6 +104,8 @@ public class StripedDataStreamer extends DataStreamer { private final Coordinator coordinator; private final int index; private volatile boolean failed; + private final ECSchema schema; + private final int cellSize; StripedDataStreamer(HdfsFileStatus stat, DFSClient dfsClient, String src, @@ -120,6 +117,8 @@ public class StripedDataStreamer extends DataStreamer { byteArrayManage, favoredNodes); this.index = index; this.coordinator = coordinator; + this.schema = stat.getErasureCodingPolicy().getSchema(); + this.cellSize = stat.getErasureCodingPolicy().getCellSize(); } int getIndex() { @@ -135,7 +134,7 @@ public class StripedDataStreamer extends DataStreamer { } private boolean isParityStreamer() { - return index >= NUM_DATA_BLOCKS; + return index >= schema.getNumDataUnits(); } @Override @@ -168,7 +167,7 @@ public class StripedDataStreamer extends DataStreamer { if (block != null) { // set numByte for the previous block group long bytes = 0; - for (int i = 0; i < NUM_DATA_BLOCKS; i++) { + for (int i = 0; i < schema.getNumDataUnits(); i++) { final ExtendedBlock b = coordinator.takeEndBlock(i); StripedBlockUtil.checkBlocks(index, block, i, b); bytes += b.getNumBytes(); @@ -183,15 +182,15 @@ public class StripedDataStreamer extends DataStreamer { final LocatedBlock lb = StripedDataStreamer.super.locateFollowingBlock( excludedNodes); - if (lb.getLocations().length < HdfsConstants.NUM_DATA_BLOCKS) { + if (lb.getLocations().length < schema.getNumDataUnits()) { throw new IOException( "Failed to get datablocks number of nodes from namenode: blockGroupSize= " - + (HdfsConstants.NUM_DATA_BLOCKS + HdfsConstants.NUM_PARITY_BLOCKS) + + (schema.getNumDataUnits() + schema.getNumParityUnits()) + ", blocks.length= " + lb.getLocations().length); } - final LocatedBlock[] blocks = StripedBlockUtil.parseStripedBlockGroup( - (LocatedStripedBlock)lb, - BLOCK_STRIPED_CELL_SIZE, NUM_DATA_BLOCKS, NUM_PARITY_BLOCKS); + final LocatedBlock[] blocks = + StripedBlockUtil.parseStripedBlockGroup((LocatedStripedBlock) lb, + cellSize, schema.getNumDataUnits(), schema.getNumParityUnits()); for (int i = 0; i < blocks.length; i++) { StripedDataStreamer si = coordinator.getStripedDataStreamer(i); @@ -233,9 +232,10 @@ public class StripedDataStreamer extends DataStreamer { final LocatedBlock updated = callUpdateBlockForPipeline(bg); final long newGS = updated.getBlock().getGenerationStamp(); final LocatedBlock[] updatedBlks = StripedBlockUtil - .parseStripedBlockGroup((LocatedStripedBlock) updated, - BLOCK_STRIPED_CELL_SIZE, NUM_DATA_BLOCKS, NUM_PARITY_BLOCKS); - for (int i = 0; i < NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS; i++) { + .parseStripedBlockGroup((LocatedStripedBlock) updated, cellSize, + schema.getNumDataUnits(), schema.getNumParityUnits()); + for (int i = 0; i < schema.getNumDataUnits() + + schema.getNumParityUnits(); i++) { StripedDataStreamer si = coordinator.getStripedDataStreamer(i); if (si.isFailed()) { continue; // skipping failed data streamer @@ -280,7 +280,7 @@ public class StripedDataStreamer extends DataStreamer { final ExtendedBlock bg = coordinator.getBlockGroup(); final ExtendedBlock newBG = newBlock(bg, newGS); - final int n = NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS; + final int n = schema.getNumDataUnits() + schema.getNumParityUnits(); final DatanodeInfo[] newNodes = new DatanodeInfo[n]; final String[] newStorageIDs = new String[n]; for (int i = 0; i < n; i++) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java index ece998488e3..524248cc325 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java @@ -192,6 +192,7 @@ public class PBHelper { StripedBlockWithLocations sblk = (StripedBlockWithLocations) blk; builder.setIndices(PBHelperClient.getByteString(sblk.getIndices())); builder.setDataBlockNum(sblk.getDataBlockNum()); + builder.setCellSize(sblk.getCellSize()); } return builder.build(); } @@ -207,7 +208,7 @@ public class PBHelper { PBHelperClient.convertStorageTypes(storageTypes, storageUuids.size())); if (b.hasIndices()) { blk = new StripedBlockWithLocations(blk, b.getIndices().toByteArray(), - (short) b.getDataBlockNum()); + (short) b.getDataBlockNum(), b.getCellSize()); } return blk; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java index 07c3c01cfc7..4c4fc275584 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java @@ -415,11 +415,14 @@ public class Dispatcher { final byte[] indices; final short dataBlockNum; + final int cellSize; - public DBlockStriped(Block block, byte[] indices, short dataBlockNum) { + public DBlockStriped(Block block, byte[] indices, short dataBlockNum, + int cellSize) { super(block); this.indices = indices; this.dataBlockNum = dataBlockNum; + this.cellSize = cellSize; } public DBlock getInternalBlock(StorageGroup storage) { @@ -429,8 +432,8 @@ public class Dispatcher { } byte idxInGroup = indices[idxInLocs]; long blkId = getBlock().getBlockId() + idxInGroup; - long numBytes = getInternalBlockLength(getNumBytes(), - HdfsConstants.BLOCK_STRIPED_CELL_SIZE, dataBlockNum, idxInGroup); + long numBytes = getInternalBlockLength(getNumBytes(), cellSize, + dataBlockNum, idxInGroup); Block blk = new Block(getBlock()); blk.setBlockId(blkId); blk.setNumBytes(numBytes); @@ -717,8 +720,8 @@ public class Dispatcher { bytesReceived += sblkLocs.getBlock().getNumBytes() / sblkLocs.getDataBlockNum(); block = new DBlockStriped(sblkLocs.getBlock(), sblkLocs.getIndices(), - sblkLocs.getDataBlockNum()); - } else{ + sblkLocs.getDataBlockNum(), sblkLocs.getCellSize()); + } else { bytesReceived += blkLocs.getBlock().getNumBytes(); block = new DBlock(blkLocs.getBlock()); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java index df48655009c..d1067b3e037 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java @@ -22,8 +22,6 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState; import org.apache.hadoop.hdfs.util.StripedBlockUtil; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; -import static org.apache.hadoop.hdfs.protocol.HdfsConstants.BLOCK_STRIPED_CELL_SIZE; - /** * Subclass of {@link BlockInfo}, presenting a block group in erasure coding. * @@ -65,6 +63,10 @@ public class BlockInfoStriped extends BlockInfo { return (short) ecPolicy.getNumParityUnits(); } + public int getCellSize() { + return ecPolicy.getCellSize(); + } + /** * If the block is committed/completed and its length is less than a full * stripe, it returns the the number of actual data blocks. @@ -73,7 +75,7 @@ public class BlockInfoStriped extends BlockInfo { public short getRealDataBlockNum() { if (isComplete() || getBlockUCState() == BlockUCState.COMMITTED) { return (short) Math.min(getDataBlockNum(), - (getNumBytes() - 1) / BLOCK_STRIPED_CELL_SIZE + 1); + (getNumBytes() - 1) / ecPolicy.getCellSize() + 1); } else { return getDataBlockNum(); } @@ -200,7 +202,7 @@ public class BlockInfoStriped extends BlockInfo { // `getNumBytes` is the total of actual data block size. return StripedBlockUtil.spaceConsumedByStripedBlock(getNumBytes(), ecPolicy.getNumDataUnits(), ecPolicy.getNumParityUnits(), - BLOCK_STRIPED_CELL_SIZE); + ecPolicy.getCellSize()); } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index e7f9262b2df..18bfc41cfbb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -92,7 +92,6 @@ import org.apache.hadoop.hdfs.util.LightWeightHashSet; import org.apache.hadoop.hdfs.util.LightWeightLinkedSet; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; -import static org.apache.hadoop.hdfs.protocol.HdfsConstants.BLOCK_STRIPED_CELL_SIZE; import static org.apache.hadoop.hdfs.util.StripedBlockUtil.getInternalBlockLength; import org.apache.hadoop.metrics2.util.MBeans; @@ -2554,10 +2553,9 @@ public class BlockManager implements BlockStatsMXBean { BlockIdManager.convertToStripedID(reported.getBlockId()); BlockInfoStriped stripedBlock = (BlockInfoStriped) storedBlock; int reportedBlkIdx = BlockIdManager.getBlockIndex(reported); - wrongSize = reported.getNumBytes() != - getInternalBlockLength(stripedBlock.getNumBytes(), - BLOCK_STRIPED_CELL_SIZE, - stripedBlock.getDataBlockNum(), reportedBlkIdx); + wrongSize = reported.getNumBytes() != getInternalBlockLength( + stripedBlock.getNumBytes(), stripedBlock.getCellSize(), + stripedBlock.getDataBlockNum(), reportedBlkIdx); } else { wrongSize = storedBlock.getNumBytes() != reported.getNumBytes(); } @@ -3413,7 +3411,7 @@ public class BlockManager implements BlockStatsMXBean { (byte) blockStriped.getStorageBlockIndex(locations.get(i)); } results.add(new StripedBlockWithLocations(blkWithLocs, indices, - blockStriped.getDataBlockNum())); + blockStriped.getDataBlockNum(), blockStriped.getCellSize())); // approximate size return block.getNumBytes() / blockStriped.getDataBlockNum(); }else{ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java index 34d92d0d7f1..1e07cad69b6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java @@ -185,7 +185,8 @@ public class Mover { for (int i = 0; i < indices.length; i++) { indices[i] = (byte) lsb.getBlockIndices()[i]; } - db = new DBlockStriped(blk, indices, (short) ecPolicy.getNumDataUnits()); + db = new DBlockStriped(blk, indices, (short) ecPolicy.getNumDataUnits(), + ecPolicy.getCellSize()); } else { db = new DBlock(blk); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java index 867b4515469..cfc939ef1d5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java @@ -204,9 +204,14 @@ class FSDirWriteFileOp { clientMachine = pendingFile.getFileUnderConstructionFeature() .getClientMachine(); isStriped = pendingFile.isStriped(); - numTargets = isStriped ? - HdfsConstants.NUM_DATA_BLOCKS + HdfsConstants.NUM_PARITY_BLOCKS : - pendingFile.getFileReplication(); + ErasureCodingPolicy ecPolicy = null; + if (isStriped) { + ecPolicy = FSDirErasureCodingOp.getErasureCodingPolicy(fsn, src); + numTargets = (short) (ecPolicy.getSchema().getNumDataUnits() + + ecPolicy.getSchema().getNumParityUnits()); + } else { + numTargets = pendingFile.getFileReplication(); + } storagePolicyID = pendingFile.getStoragePolicyID(); return new ValidateAddBlockResult(blockSize, numTargets, storagePolicyID, clientMachine, isStriped); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java index c765edce714..242c456bdb5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java @@ -34,7 +34,7 @@ import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.QuotaExceededException; import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; @@ -388,8 +388,13 @@ public class INodeFile extends INodeWithAdditionalFields } max = maxInSnapshot > max ? maxInSnapshot : max; } - return isStriped() ? - HdfsConstants.NUM_DATA_BLOCKS + HdfsConstants.NUM_PARITY_BLOCKS : max; + if(!isStriped()){ + return max; + } + // TODO support more policies based on policyId + ErasureCodingPolicy ecPolicy = + ErasureCodingPolicyManager.getSystemDefaultPolicy(); + return (short) (ecPolicy.getNumDataUnits() + ecPolicy.getNumParityUnits()); } /** Set the replication factor of this file. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java index 0507faf1af1..b6eddb61579 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java @@ -95,16 +95,17 @@ public class BlocksWithLocations { public static class StripedBlockWithLocations extends BlockWithLocations { final byte[] indices; final short dataBlockNum; + final int cellSize; public StripedBlockWithLocations(BlockWithLocations blk, byte[] indices, - short dataBlockNum) { + short dataBlockNum, int cellSize) { super(blk.getBlock(), blk.getDatanodeUuids(), blk.getStorageIDs(), blk.getStorageTypes()); Preconditions.checkArgument( blk.getDatanodeUuids().length == indices.length); this.indices = indices; this.dataBlockNum = dataBlockNum; - + this.cellSize = cellSize; } public byte[] getIndices() { @@ -114,6 +115,10 @@ public class BlocksWithLocations { public short getDataBlockNum() { return dataBlockNum; } + + public int getCellSize() { + return cellSize; + } } private final BlockWithLocations[] blocks; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java index 12d48114873..274d3195fb9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java @@ -166,8 +166,8 @@ import org.junit.Assume; import org.mockito.internal.util.reflection.Whitebox; import com.google.common.annotations.VisibleForTesting; -import static org.apache.hadoop.hdfs.protocol.HdfsConstants.BLOCK_STRIPED_CELL_SIZE; -import static org.apache.hadoop.hdfs.protocol.HdfsConstants.NUM_DATA_BLOCKS; +import static org.apache.hadoop.hdfs.StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE; +import static org.apache.hadoop.hdfs.StripedFileTestUtil.NUM_DATA_BLOCKS; /** Utilities for HDFS tests */ public class DFSTestUtil { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java index ded86e7fb38..8d4a0cf5c9b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java @@ -25,7 +25,6 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.web.ByteRangeInputStream; @@ -38,18 +37,19 @@ import java.util.Random; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; -import static org.apache.hadoop.hdfs.protocol.HdfsConstants.BLOCK_STRIPED_CELL_SIZE; - public class StripedFileTestUtil { public static final Log LOG = LogFactory.getLog(StripedFileTestUtil.class); + /* + * These values correspond to the values used by the system default erasure + * coding policy. + */ + public static final short NUM_DATA_BLOCKS = (short) 6; + public static final short NUM_PARITY_BLOCKS = (short) 3; + public static final int BLOCK_STRIPED_CELL_SIZE = 64 * 1024; - static int dataBlocks = HdfsConstants.NUM_DATA_BLOCKS; - static int parityBlocks = HdfsConstants.NUM_PARITY_BLOCKS; - - static final int cellSize = HdfsConstants.BLOCK_STRIPED_CELL_SIZE; static final int stripesPerBlock = 4; - static final int blockSize = cellSize * stripesPerBlock; - static final int numDNs = dataBlocks + parityBlocks + 2; + static final int blockSize = BLOCK_STRIPED_CELL_SIZE * stripesPerBlock; + static final int numDNs = NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS + 2; static final Random random = new Random(); @@ -85,9 +85,9 @@ public class StripedFileTestUtil { static void verifyPread(FileSystem fs, Path srcPath, int fileLength, byte[] expected, byte[] buf) throws IOException { try (FSDataInputStream in = fs.open(srcPath)) { - int[] startOffsets = {0, 1, cellSize - 102, cellSize, cellSize + 102, - cellSize * (dataBlocks - 1), cellSize * (dataBlocks - 1) + 102, - cellSize * dataBlocks, fileLength - 102, fileLength - 1}; + int[] startOffsets = {0, 1, BLOCK_STRIPED_CELL_SIZE - 102, BLOCK_STRIPED_CELL_SIZE, BLOCK_STRIPED_CELL_SIZE + 102, + BLOCK_STRIPED_CELL_SIZE * (NUM_DATA_BLOCKS - 1), BLOCK_STRIPED_CELL_SIZE * (NUM_DATA_BLOCKS - 1) + 102, + BLOCK_STRIPED_CELL_SIZE * NUM_DATA_BLOCKS, fileLength - 102, fileLength - 1}; for (int startOffset : startOffsets) { startOffset = Math.max(0, Math.min(startOffset, fileLength - 1)); int remaining = fileLength - startOffset; @@ -158,21 +158,21 @@ public class StripedFileTestUtil { pos = 0; assertSeekAndRead(in, pos, fileLength); - if (fileLength > cellSize) { + if (fileLength > BLOCK_STRIPED_CELL_SIZE) { // seek to cellSize boundary - pos = cellSize - 1; + pos = BLOCK_STRIPED_CELL_SIZE - 1; assertSeekAndRead(in, pos, fileLength); } - if (fileLength > cellSize * dataBlocks) { + if (fileLength > BLOCK_STRIPED_CELL_SIZE * NUM_DATA_BLOCKS) { // seek to striped cell group boundary - pos = cellSize * dataBlocks - 1; + pos = BLOCK_STRIPED_CELL_SIZE * NUM_DATA_BLOCKS - 1; assertSeekAndRead(in, pos, fileLength); } - if (fileLength > blockSize * dataBlocks) { + if (fileLength > blockSize * NUM_DATA_BLOCKS) { // seek to striped block group boundary - pos = blockSize * dataBlocks - 1; + pos = blockSize * NUM_DATA_BLOCKS - 1; assertSeekAndRead(in, pos, fileLength); } @@ -235,12 +235,12 @@ public class StripedFileTestUtil { * number of actual data internal blocks. Otherwise returns NUM_DATA_BLOCKS. */ public static short getRealDataBlockNum(int numBytes) { - return (short) Math.min(dataBlocks, + return (short) Math.min(NUM_DATA_BLOCKS, (numBytes - 1) / BLOCK_STRIPED_CELL_SIZE + 1); } public static short getRealTotalBlockNum(int numBytes) { - return (short) (getRealDataBlockNum(numBytes) + parityBlocks); + return (short) (getRealDataBlockNum(numBytes) + NUM_PARITY_BLOCKS); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java index 6fcf644dd8b..549a744bc7d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java @@ -24,7 +24,6 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.protocol.Block; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock; @@ -60,9 +59,9 @@ public class TestDFSStripedInputStream { private final Path dirPath = new Path("/striped"); private Path filePath = new Path(dirPath, "file"); private final ErasureCodingPolicy ecPolicy = ErasureCodingPolicyManager.getSystemDefaultPolicy(); - private final short DATA_BLK_NUM = HdfsConstants.NUM_DATA_BLOCKS; - private final short PARITY_BLK_NUM = HdfsConstants.NUM_PARITY_BLOCKS; - private final int CELLSIZE = HdfsConstants.BLOCK_STRIPED_CELL_SIZE; + private final short DATA_BLK_NUM = StripedFileTestUtil.NUM_DATA_BLOCKS; + private final short PARITY_BLK_NUM = StripedFileTestUtil.NUM_PARITY_BLOCKS; + private final int CELLSIZE = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE; private final int NUM_STRIPE_PER_BLOCK = 2; private final int INTERNAL_BLOCK_SIZE = NUM_STRIPE_PER_BLOCK * CELLSIZE; private final int BLOCK_GROUP_SIZE = DATA_BLK_NUM * INTERNAL_BLOCK_SIZE; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java index 4263ffaf289..0641e8ea65e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java @@ -28,7 +28,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock; @@ -51,13 +50,13 @@ public class TestDFSStripedOutputStream { GenericTestUtils.setLogLevel(DataStreamer.LOG, Level.ALL); } - private int dataBlocks = HdfsConstants.NUM_DATA_BLOCKS; - private int parityBlocks = HdfsConstants.NUM_PARITY_BLOCKS; + private int dataBlocks = StripedFileTestUtil.NUM_DATA_BLOCKS; + private int parityBlocks = StripedFileTestUtil.NUM_PARITY_BLOCKS; private MiniDFSCluster cluster; private DistributedFileSystem fs; private Configuration conf; - private final int cellSize = HdfsConstants.BLOCK_STRIPED_CELL_SIZE; + private final int cellSize = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE; private final int stripesPerBlock = 4; private final int blockSize = cellSize * stripesPerBlock; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java index afb7f95cbb0..44a29e671e5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java @@ -65,9 +65,9 @@ public class TestDFSStripedOutputStreamWithFailure { GenericTestUtils.setLogLevel(DataStreamer.LOG, Level.ALL); } - private static final int NUM_DATA_BLOCKS = HdfsConstants.NUM_DATA_BLOCKS; - private static final int NUM_PARITY_BLOCKS = HdfsConstants.NUM_PARITY_BLOCKS; - private static final int CELL_SIZE = HdfsConstants.BLOCK_STRIPED_CELL_SIZE; + private static final int NUM_DATA_BLOCKS = StripedFileTestUtil.NUM_DATA_BLOCKS; + private static final int NUM_PARITY_BLOCKS = StripedFileTestUtil.NUM_PARITY_BLOCKS; + private static final int CELL_SIZE = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE; private static final int STRIPES_PER_BLOCK = 4; private static final int BLOCK_SIZE = CELL_SIZE * STRIPES_PER_BLOCK; private static final int BLOCK_GROUP_SIZE = BLOCK_SIZE * NUM_DATA_BLOCKS; @@ -255,7 +255,7 @@ public class TestDFSStripedOutputStreamWithFailure { DatanodeReportType.LIVE); assertEquals("Mismatches number of live Dns ", numDatanodes, info.length); Path srcPath = new Path(dir, "testAddBlockWhenNoSufficientParityNodes"); - int fileLength = HdfsConstants.BLOCK_STRIPED_CELL_SIZE - 1000; + int fileLength = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE - 1000; final byte[] expected = StripedFileTestUtil.generateBytes(fileLength); DFSTestUtil.writeFile(dfs, srcPath, new String(expected)); StripedFileTestUtil.verifySeek(dfs, srcPath, fileLength); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java index 68cd25edeac..9cf970b7f4d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java @@ -46,16 +46,16 @@ import java.io.IOException; import java.nio.ByteBuffer; import static org.apache.hadoop.hdfs.StripedFileTestUtil.blockSize; -import static org.apache.hadoop.hdfs.StripedFileTestUtil.cellSize; -import static org.apache.hadoop.hdfs.StripedFileTestUtil.dataBlocks; import static org.apache.hadoop.hdfs.StripedFileTestUtil.numDNs; -import static org.apache.hadoop.hdfs.StripedFileTestUtil.parityBlocks; public class TestReadStripedFileWithDecoding { static final Log LOG = LogFactory.getLog(TestReadStripedFileWithDecoding.class); private MiniDFSCluster cluster; private DistributedFileSystem fs; + private final short dataBlocks = StripedFileTestUtil.NUM_DATA_BLOCKS; + private final short parityBlocks = StripedFileTestUtil.NUM_PARITY_BLOCKS; + private final int cellSize = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE; private final int smallFileLength = blockSize * dataBlocks - 123; private final int largeFileLength = blockSize * dataBlocks + 123; private final int[] fileLengths = {smallFileLength, largeFileLength}; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java index ca376afa4a4..f521d8edb18 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java @@ -32,8 +32,6 @@ import org.junit.Test; import java.io.IOException; import static org.apache.hadoop.hdfs.StripedFileTestUtil.blockSize; -import static org.apache.hadoop.hdfs.StripedFileTestUtil.cellSize; -import static org.apache.hadoop.hdfs.StripedFileTestUtil.dataBlocks; import static org.apache.hadoop.hdfs.StripedFileTestUtil.numDNs; /** @@ -46,6 +44,8 @@ public class TestReadStripedFileWithMissingBlocks { private static MiniDFSCluster cluster; private static DistributedFileSystem fs; private static Configuration conf = new HdfsConfiguration(); + private final short dataBlocks = StripedFileTestUtil.NUM_DATA_BLOCKS; + private final int cellSize = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE; private final int fileLength = blockSize * dataBlocks + 123; @Before @@ -112,7 +112,7 @@ public class TestReadStripedFileWithMissingBlocks { } for (int i = 0; i < missingParityNum; i++) { missingDataNodes[i + missingDataNum] = i + - Math.min(StripedFileTestUtil.dataBlocks, dataBlocks); + Math.min(StripedFileTestUtil.NUM_DATA_BLOCKS, dataBlocks); } stopDataNodes(locs, missingDataNodes); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRecoverStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRecoverStripedFile.java index ed2ca7b367d..75dc6a02ac4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRecoverStripedFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRecoverStripedFile.java @@ -58,9 +58,9 @@ import org.junit.Test; public class TestRecoverStripedFile { public static final Log LOG = LogFactory.getLog(TestRecoverStripedFile.class); - private static final int dataBlkNum = HdfsConstants.NUM_DATA_BLOCKS; - private static final int parityBlkNum = HdfsConstants.NUM_PARITY_BLOCKS; - private static final int cellSize = HdfsConstants.BLOCK_STRIPED_CELL_SIZE; + private static final int dataBlkNum = StripedFileTestUtil.NUM_DATA_BLOCKS; + private static final int parityBlkNum = StripedFileTestUtil.NUM_PARITY_BLOCKS; + private static final int cellSize = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE; private static final int blockSize = cellSize * 3; private static final int groupSize = dataBlkNum + parityBlkNum; private static final int dnNum = groupSize + parityBlkNum; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java index 9ab0834a614..9853b8a59fd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java @@ -40,10 +40,10 @@ import static org.junit.Assert.assertTrue; public class TestSafeModeWithStripedFile { - static final short DATA_BLK_NUM = HdfsConstants.NUM_DATA_BLOCKS; - static final short PARITY_BLK_NUM = HdfsConstants.NUM_PARITY_BLOCKS; + static final short DATA_BLK_NUM = StripedFileTestUtil.NUM_DATA_BLOCKS; + static final short PARITY_BLK_NUM = StripedFileTestUtil.NUM_PARITY_BLOCKS; static final int numDNs = DATA_BLK_NUM + PARITY_BLK_NUM; - static final int cellSize = HdfsConstants.BLOCK_STRIPED_CELL_SIZE; + static final int cellSize = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE; static final int blockSize = cellSize * 2; static MiniDFSCluster cluster; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java index 5381fcabeab..3d51f0c725c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java @@ -37,8 +37,6 @@ import java.io.IOException; import java.nio.ByteBuffer; import static org.apache.hadoop.hdfs.StripedFileTestUtil.blockSize; -import static org.apache.hadoop.hdfs.StripedFileTestUtil.cellSize; -import static org.apache.hadoop.hdfs.StripedFileTestUtil.dataBlocks; import static org.apache.hadoop.hdfs.StripedFileTestUtil.numDNs; import static org.apache.hadoop.hdfs.StripedFileTestUtil.stripesPerBlock; @@ -46,6 +44,8 @@ public class TestWriteReadStripedFile { public static final Log LOG = LogFactory.getLog(TestWriteReadStripedFile.class); private static MiniDFSCluster cluster; private static DistributedFileSystem fs; + private static int cellSize = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE; + private static short dataBlocks = StripedFileTestUtil.NUM_DATA_BLOCKS; private static Configuration conf = new HdfsConfiguration(); static { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java index c28bff82d2a..c0dca4e8662 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java @@ -31,9 +31,7 @@ import java.io.IOException; import java.util.concurrent.atomic.AtomicInteger; import static org.apache.hadoop.hdfs.StripedFileTestUtil.blockSize; -import static org.apache.hadoop.hdfs.StripedFileTestUtil.dataBlocks; import static org.apache.hadoop.hdfs.StripedFileTestUtil.numDNs; -import static org.apache.hadoop.hdfs.StripedFileTestUtil.parityBlocks; public class TestWriteStripedFileWithFailure { public static final Log LOG = LogFactory @@ -41,6 +39,8 @@ public class TestWriteStripedFileWithFailure { private static MiniDFSCluster cluster; private static FileSystem fs; private static Configuration conf = new HdfsConfiguration(); + private final short dataBlocks = StripedFileTestUtil.NUM_DATA_BLOCKS; + private final short parityBlocks = StripedFileTestUtil.NUM_PARITY_BLOCKS; private final int smallFileLength = blockSize * dataBlocks - 123; private final int largeFileLength = blockSize * dataBlocks + 123; private final int[] fileLengths = {smallFileLength, largeFileLength}; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java index 00a4575253d..0d51ac899b3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java @@ -34,6 +34,7 @@ import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; @@ -201,7 +202,8 @@ public class TestPBHelper { BlockWithLocations blkLocs = new BlockWithLocations(new Block(bid, 0, 1), datanodeUuids, storageIDs, storageTypes); if (isStriped) { - blkLocs = new StripedBlockWithLocations(blkLocs, indices, dataBlkNum); + blkLocs = new StripedBlockWithLocations(blkLocs, indices, dataBlkNum, + StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE); } return blkLocs; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java index 25012e74066..124bf8060ed 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java @@ -67,6 +67,7 @@ import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.NameNodeProxies; +import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; @@ -149,10 +150,10 @@ public class TestBalancer { conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 1L); } - int dataBlocks = HdfsConstants.NUM_DATA_BLOCKS; - int parityBlocks = HdfsConstants.NUM_PARITY_BLOCKS; + int dataBlocks = StripedFileTestUtil.NUM_DATA_BLOCKS; + int parityBlocks = StripedFileTestUtil.NUM_PARITY_BLOCKS; int groupSize = dataBlocks + parityBlocks; - private final static int cellSize = HdfsConstants.BLOCK_STRIPED_CELL_SIZE; + private final static int cellSize = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE; private final static int stripesPerBlock = 4; static int DEFAULT_STRIPE_BLOCK_SIZE = cellSize * stripesPerBlock; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java index 6cc1dcd44fa..4650dce6ed4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java @@ -31,8 +31,8 @@ import java.io.DataOutputStream; import java.io.ByteArrayOutputStream; import java.nio.ByteBuffer; -import static org.apache.hadoop.hdfs.protocol.HdfsConstants.NUM_DATA_BLOCKS; -import static org.apache.hadoop.hdfs.protocol.HdfsConstants.NUM_PARITY_BLOCKS; +import static org.apache.hadoop.hdfs.StripedFileTestUtil.NUM_DATA_BLOCKS; +import static org.apache.hadoop.hdfs.StripedFileTestUtil.NUM_PARITY_BLOCKS; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java index 9dc537c4b69..ae33ffe3401 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java @@ -20,7 +20,7 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock; import org.apache.hadoop.hdfs.server.balancer.TestBalancer; @@ -31,9 +31,9 @@ import java.io.IOException; public class TestBlockTokenWithDFSStriped extends TestBlockTokenWithDFS { - private final static int dataBlocks = HdfsConstants.NUM_DATA_BLOCKS; - private final static int parityBlocks = HdfsConstants.NUM_PARITY_BLOCKS; - private final static int cellSize = HdfsConstants.BLOCK_STRIPED_CELL_SIZE; + private final static int dataBlocks = StripedFileTestUtil.NUM_DATA_BLOCKS; + private final static int parityBlocks = StripedFileTestUtil.NUM_PARITY_BLOCKS; + private final static int cellSize = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE; private final static int stripesPerBlock = 4; private final static int numDNs = dataBlocks + parityBlocks + 2; private static MiniDFSCluster cluster; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java index 9f4c47df858..00b4e3bba37 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java @@ -37,7 +37,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.test.GenericTestUtils; @@ -58,9 +58,9 @@ public class TestSequentialBlockGroupId { private final short REPLICATION = 1; private final long SEED = 0; - private final int dataBlocks = HdfsConstants.NUM_DATA_BLOCKS; - private final int parityBlocks = HdfsConstants.NUM_PARITY_BLOCKS; - private final int cellSize = HdfsConstants.BLOCK_STRIPED_CELL_SIZE; + private final int dataBlocks = StripedFileTestUtil.NUM_DATA_BLOCKS; + private final int parityBlocks = StripedFileTestUtil.NUM_PARITY_BLOCKS; + private final int cellSize = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE; private final int stripesPerBlock = 2; private final int blockSize = cellSize * stripesPerBlock; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java index 7794f04ecd5..3a9748f2239 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.MiniDFSNNTopology; import org.apache.hadoop.hdfs.NameNodeProxies; +import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.LocatedBlock; @@ -415,9 +416,9 @@ public class TestMover { } } - int dataBlocks = HdfsConstants.NUM_DATA_BLOCKS; - int parityBlocks = HdfsConstants.NUM_PARITY_BLOCKS; - private final static int cellSize = HdfsConstants.BLOCK_STRIPED_CELL_SIZE; + int dataBlocks = StripedFileTestUtil.NUM_DATA_BLOCKS; + int parityBlocks = StripedFileTestUtil.NUM_PARITY_BLOCKS; + private final static int cellSize = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE; private final static int stripesPerBlock = 4; static int DEFAULT_STRIPE_BLOCK_SIZE = cellSize * stripesPerBlock; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java index ae2cbab1688..64d33a406fa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.HdfsConstants; @@ -49,10 +50,10 @@ public class TestAddOverReplicatedStripedBlocks { private DistributedFileSystem fs; private final Path dirPath = new Path("/striped"); private Path filePath = new Path(dirPath, "file"); - private final short DATA_BLK_NUM = HdfsConstants.NUM_DATA_BLOCKS; - private final short PARITY_BLK_NUM = HdfsConstants.NUM_PARITY_BLOCKS; - private final short GROUP_SIZE = DATA_BLK_NUM + PARITY_BLK_NUM; - private final int CELLSIZE = HdfsConstants.BLOCK_STRIPED_CELL_SIZE; + private final short DATA_BLK_NUM = StripedFileTestUtil.NUM_DATA_BLOCKS; + private final short PARITY_BLK_NUM = StripedFileTestUtil.NUM_PARITY_BLOCKS; + private final short GROUP_SIZE = (short) (DATA_BLK_NUM + PARITY_BLK_NUM); + private final int CELLSIZE = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE; private final int NUM_STRIPE_PER_BLOCK = 4; private final int BLOCK_SIZE = NUM_STRIPE_PER_BLOCK * CELLSIZE; private final int numDNs = GROUP_SIZE + 3; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java index 4351fb9f3ab..836bff1b4a7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; import org.apache.hadoop.hdfs.protocol.DatanodeID; @@ -57,13 +58,13 @@ import java.util.List; import java.util.UUID; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT; -import static org.apache.hadoop.hdfs.protocol.HdfsConstants.BLOCK_STRIPED_CELL_SIZE; -import static org.apache.hadoop.hdfs.protocol.HdfsConstants.NUM_DATA_BLOCKS; +import static org.apache.hadoop.hdfs.StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE; +import static org.apache.hadoop.hdfs.StripedFileTestUtil.NUM_DATA_BLOCKS; import static org.junit.Assert.assertEquals; public class TestAddStripedBlocks { - private final short GROUP_SIZE = HdfsConstants.NUM_DATA_BLOCKS + - HdfsConstants.NUM_PARITY_BLOCKS; + private final short GROUP_SIZE = (short) (StripedFileTestUtil.NUM_DATA_BLOCKS + + StripedFileTestUtil.NUM_PARITY_BLOCKS); private MiniDFSCluster cluster; private DistributedFileSystem dfs; @@ -163,8 +164,8 @@ public class TestAddStripedBlocks { boolean checkReplica) { assertEquals(0, block.numNodes()); Assert.assertFalse(block.isComplete()); - Assert.assertEquals(HdfsConstants.NUM_DATA_BLOCKS, block.getDataBlockNum()); - Assert.assertEquals(HdfsConstants.NUM_PARITY_BLOCKS, + Assert.assertEquals(StripedFileTestUtil.NUM_DATA_BLOCKS, block.getDataBlockNum()); + Assert.assertEquals(StripedFileTestUtil.NUM_PARITY_BLOCKS, block.getParityBlockNum()); Assert.assertEquals(0, block.getBlockId() & HdfsServerConstants.BLOCK_GROUP_INDEX_MASK); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java index 5298980f84d..917b0f2e127 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java @@ -45,6 +45,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; @@ -450,8 +451,8 @@ public class TestFSEditLogLoader { long blkId = 1; long blkNumBytes = 1024; long timestamp = 1426222918; - short blockNum = HdfsConstants.NUM_DATA_BLOCKS; - short parityNum = HdfsConstants.NUM_PARITY_BLOCKS; + short blockNum = StripedFileTestUtil.NUM_DATA_BLOCKS; + short parityNum = StripedFileTestUtil.NUM_PARITY_BLOCKS; //set the storage policy of the directory fs.mkdir(new Path(testDir), new FsPermission("755")); @@ -522,8 +523,8 @@ public class TestFSEditLogLoader { long blkId = 1; long blkNumBytes = 1024; long timestamp = 1426222918; - short blockNum = HdfsConstants.NUM_DATA_BLOCKS; - short parityNum = HdfsConstants.NUM_PARITY_BLOCKS; + short blockNum = StripedFileTestUtil.NUM_DATA_BLOCKS; + short parityNum = StripedFileTestUtil.NUM_PARITY_BLOCKS; //set the storage policy of the directory fs.mkdir(new Path(testDir), new FsPermission("755")); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java index 7565e86d3d1..e45d08d6c58 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java @@ -52,6 +52,7 @@ import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.client.HdfsDataOutputStream.SyncFlag; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; @@ -414,8 +415,8 @@ public class TestFSImage { */ @Test public void testSupportBlockGroup() throws IOException { - final short GROUP_SIZE = HdfsConstants.NUM_DATA_BLOCKS + - HdfsConstants.NUM_PARITY_BLOCKS; + final short GROUP_SIZE = (short) (StripedFileTestUtil.NUM_DATA_BLOCKS + + StripedFileTestUtil.NUM_PARITY_BLOCKS); final int BLOCK_SIZE = 8 * 1024 * 1024; Configuration conf = new HdfsConfiguration(); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE); @@ -447,8 +448,8 @@ public class TestFSImage { BlockInfo[] blks = inode.getBlocks(); assertEquals(1, blks.length); assertTrue(blks[0].isStriped()); - assertEquals(HdfsConstants.NUM_DATA_BLOCKS, ((BlockInfoStriped)blks[0]).getDataBlockNum()); - assertEquals(HdfsConstants.NUM_PARITY_BLOCKS, ((BlockInfoStriped)blks[0]).getParityBlockNum()); + assertEquals(StripedFileTestUtil.NUM_DATA_BLOCKS, ((BlockInfoStriped)blks[0]).getDataBlockNum()); + assertEquals(StripedFileTestUtil.NUM_PARITY_BLOCKS, ((BlockInfoStriped)blks[0]).getParityBlockNum()); } finally { cluster.shutdown(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java index efa5027fc61..26f9b8ee4f7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; @@ -113,9 +114,9 @@ public class TestQuotaWithStripedBlocks { final long actualDiskUsed = dirNode.getDirectoryWithQuotaFeature() .getSpaceConsumed().getTypeSpaces().get(StorageType.DISK); // In this case the file's real size is cell size * block group size. - Assert.assertEquals(HdfsConstants.BLOCK_STRIPED_CELL_SIZE * GROUP_SIZE, + Assert.assertEquals(StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE * GROUP_SIZE, actualSpaceUsed); - Assert.assertEquals(HdfsConstants.BLOCK_STRIPED_CELL_SIZE * GROUP_SIZE, + Assert.assertEquals(StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE * GROUP_SIZE, actualDiskUsed); } finally { IOUtils.cleanup(null, out); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestRecoverStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestRecoverStripedBlocks.java index 3134373dab4..6774aedba33 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestRecoverStripedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestRecoverStripedBlocks.java @@ -35,16 +35,16 @@ import org.apache.hadoop.hdfs.server.protocol.BlockECRecoveryCommand.BlockECReco import org.junit.Test; import java.util.List; -import static org.apache.hadoop.hdfs.protocol.HdfsConstants.BLOCK_STRIPED_CELL_SIZE; -import static org.apache.hadoop.hdfs.protocol.HdfsConstants.NUM_DATA_BLOCKS; -import static org.apache.hadoop.hdfs.protocol.HdfsConstants.NUM_PARITY_BLOCKS; +import static org.apache.hadoop.hdfs.StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE; +import static org.apache.hadoop.hdfs.StripedFileTestUtil.NUM_DATA_BLOCKS; +import static org.apache.hadoop.hdfs.StripedFileTestUtil.NUM_PARITY_BLOCKS; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; public class TestRecoverStripedBlocks { private final short GROUP_SIZE = - NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS; + (short) (NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS); private MiniDFSCluster cluster; private final Path dirPath = new Path("/dir"); private Path filePath = new Path(dirPath, "file"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java index c8275822df4..9f7ea3e7bc0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; @@ -222,8 +223,8 @@ public class TestStripedINodeFile { final Path contiguousFile = new Path(parentDir, "someFile"); final DistributedFileSystem dfs; final Configuration conf = new Configuration(); - final short GROUP_SIZE = HdfsConstants.NUM_DATA_BLOCKS - + HdfsConstants.NUM_PARITY_BLOCKS; + final short GROUP_SIZE = (short) (StripedFileTestUtil.NUM_DATA_BLOCKS + + StripedFileTestUtil.NUM_PARITY_BLOCKS); conf.setInt(DFSConfigKeys.DFS_NAMENODE_MAX_XATTRS_PER_INODE_KEY, 2); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(GROUP_SIZE) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java index 1e2326a5873..6d4d79734e9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java @@ -31,7 +31,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; @@ -44,12 +44,12 @@ import org.junit.BeforeClass; import org.junit.Test; public class TestOfflineImageViewerWithStripedBlocks { - private static int dataBlocks = HdfsConstants.NUM_DATA_BLOCKS; - private static int parityBlocks = HdfsConstants.NUM_PARITY_BLOCKS; + private static int dataBlocks = StripedFileTestUtil.NUM_DATA_BLOCKS; + private static int parityBlocks = StripedFileTestUtil.NUM_PARITY_BLOCKS; private static MiniDFSCluster cluster; private static DistributedFileSystem fs; - private static final int cellSize = HdfsConstants.BLOCK_STRIPED_CELL_SIZE; + private static final int cellSize = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE; private static final int stripesPerBlock = 3; private static final int blockSize = cellSize * stripesPerBlock; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java index e61ac07e975..95b0135720f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java @@ -21,9 +21,9 @@ package org.apache.hadoop.hdfs.util; import com.google.common.base.Preconditions; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock; import org.apache.hadoop.hdfs.server.blockmanagement.BlockIdManager; @@ -77,10 +77,10 @@ import static org.junit.Assert.assertFalse; * TODO: test parity block logic */ public class TestStripedBlockUtil { - private final short DATA_BLK_NUM = HdfsConstants.NUM_DATA_BLOCKS; - private final short PARITY_BLK_NUM = HdfsConstants.NUM_PARITY_BLOCKS; - private final short BLK_GROUP_WIDTH = DATA_BLK_NUM + PARITY_BLK_NUM; - private final int CELLSIZE = HdfsConstants.BLOCK_STRIPED_CELL_SIZE; + private final short DATA_BLK_NUM = StripedFileTestUtil.NUM_DATA_BLOCKS; + private final short PARITY_BLK_NUM = StripedFileTestUtil.NUM_PARITY_BLOCKS; + private final short BLK_GROUP_WIDTH = (short) (DATA_BLK_NUM + PARITY_BLK_NUM); + private final int CELLSIZE = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE; private final int FULL_STRIPE_SIZE = DATA_BLK_NUM * CELLSIZE; /** number of full stripes in a full block group */ private final int BLK_GROUP_STRIPE_NUM = 16;