HDFS-7937. Erasure Coding: INodeFile quota computation unit tests. Contributed by Kai Sasaki.
This commit is contained in:
Jing Zhao
Zhe Zhang
parent
cd458c38a0
commit
ab76e1fe36
|
|
@ -203,8 +203,27 @@ public class BlockInfoStriped extends BlockInfo {
|
||||||
// In case striped blocks, total usage by this striped blocks should
|
// In case striped blocks, total usage by this striped blocks should
|
||||||
// be the total of data blocks and parity blocks because
|
// be the total of data blocks and parity blocks because
|
||||||
// `getNumBytes` is the total of actual data block size.
|
// `getNumBytes` is the total of actual data block size.
|
||||||
return ((getNumBytes() - 1) / (dataBlockNum * BLOCK_STRIPED_CELL_SIZE) + 1)
|
|
||||||
* BLOCK_STRIPED_CELL_SIZE * parityBlockNum + getNumBytes();
|
// 0. Calculate the total bytes per stripes <Num Bytes per Stripes>
|
||||||
|
long numBytesPerStripe = dataBlockNum * BLOCK_STRIPED_CELL_SIZE;
|
||||||
|
if (getNumBytes() % numBytesPerStripe == 0) {
|
||||||
|
return getNumBytes() / dataBlockNum * getTotalBlockNum();
|
||||||
|
}
|
||||||
|
// 1. Calculate the number of stripes in this block group. <Num Stripes>
|
||||||
|
long numStripes = (getNumBytes() - 1) / numBytesPerStripe + 1;
|
||||||
|
// 2. Calculate the parity cell length in the last stripe. Note that the
|
||||||
|
// size of parity cells should equal the size of the first cell, if it
|
||||||
|
// is not full. <Last Stripe Parity Cell Length>
|
||||||
|
long lastStripeParityCellLen = Math.min(getNumBytes() % numBytesPerStripe,
|
||||||
|
BLOCK_STRIPED_CELL_SIZE);
|
||||||
|
// 3. Total consumed space is the total of
|
||||||
|
// - The total of the full cells of data blocks and parity blocks.
|
||||||
|
// - The remaining of data block which does not make a stripe.
|
||||||
|
// - The last parity block cells. These size should be same
|
||||||
|
// to the first cell in this stripe.
|
||||||
|
return getTotalBlockNum() * (BLOCK_STRIPED_CELL_SIZE * (numStripes - 1))
|
||||||
|
+ getNumBytes() % numBytesPerStripe
|
||||||
|
+ lastStripeParityCellLen * parityBlockNum;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,229 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.hdfs.server.namenode;
|
||||||
|
|
||||||
|
import static org.junit.Assert.assertEquals;
|
||||||
|
import static org.junit.Assert.assertTrue;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
import org.apache.commons.logging.Log;
|
||||||
|
import org.apache.commons.logging.LogFactory;
|
||||||
|
import org.apache.hadoop.fs.permission.FsPermission;
|
||||||
|
import org.apache.hadoop.fs.permission.PermissionStatus;
|
||||||
|
import org.apache.hadoop.hdfs.protocol.Block;
|
||||||
|
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
|
||||||
|
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
|
||||||
|
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStripedUnderConstruction;
|
||||||
|
import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
|
||||||
|
import org.apache.hadoop.io.erasurecode.ECSchema;
|
||||||
|
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This class tests INodeFile with striped feature.
|
||||||
|
*/
|
||||||
|
public class TestStripedINodeFile {
|
||||||
|
public static final Log LOG = LogFactory.getLog(TestINodeFile.class);
|
||||||
|
|
||||||
|
private static final PermissionStatus perm = new PermissionStatus(
|
||||||
|
"userName", null, FsPermission.getDefault());
|
||||||
|
|
||||||
|
private static INodeFile createStripedINodeFile() {
|
||||||
|
return new INodeFile(INodeId.GRANDFATHER_INODE_ID, null, perm, 0L, 0L,
|
||||||
|
null, (short)0, 1024L, HdfsConstants.COLD_STORAGE_POLICY_ID);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testBlockStripedFeature()
|
||||||
|
throws IOException, InterruptedException{
|
||||||
|
INodeFile inf = createStripedINodeFile();
|
||||||
|
inf.addStripedBlocksFeature();
|
||||||
|
assertTrue(inf.isStriped());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testBlockStripedTotalBlockCount() {
|
||||||
|
ECSchema defaultSchema = ECSchemaManager.getSystemDefaultSchema();
|
||||||
|
Block blk = new Block(1);
|
||||||
|
BlockInfoStriped blockInfoStriped
|
||||||
|
= new BlockInfoStriped(blk,
|
||||||
|
(short)defaultSchema.getNumDataUnits(),
|
||||||
|
(short)defaultSchema.getNumParityUnits());
|
||||||
|
assertEquals(9, blockInfoStriped.getTotalBlockNum());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testBlockStripedLength()
|
||||||
|
throws IOException, InterruptedException {
|
||||||
|
ECSchema defaultSchema = ECSchemaManager.getSystemDefaultSchema();
|
||||||
|
INodeFile inf = createStripedINodeFile();
|
||||||
|
inf.addStripedBlocksFeature();
|
||||||
|
Block blk = new Block(1);
|
||||||
|
BlockInfoStriped blockInfoStriped
|
||||||
|
= new BlockInfoStriped(blk,
|
||||||
|
(short)defaultSchema.getNumDataUnits(),
|
||||||
|
(short)defaultSchema.getNumParityUnits());
|
||||||
|
inf.addBlock(blockInfoStriped);
|
||||||
|
assertEquals(1, inf.getBlocks().length);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testBlockStripedConsumedSpace()
|
||||||
|
throws IOException, InterruptedException {
|
||||||
|
ECSchema defaultSchema = ECSchemaManager.getSystemDefaultSchema();
|
||||||
|
INodeFile inf = createStripedINodeFile();
|
||||||
|
inf.addStripedBlocksFeature();
|
||||||
|
Block blk = new Block(1);
|
||||||
|
BlockInfoStriped blockInfoStriped
|
||||||
|
= new BlockInfoStriped(blk,
|
||||||
|
(short)defaultSchema.getNumDataUnits(),
|
||||||
|
(short)defaultSchema.getNumParityUnits());
|
||||||
|
blockInfoStriped.setNumBytes(1);
|
||||||
|
inf.addBlock(blockInfoStriped);
|
||||||
|
// 0. Calculate the total bytes per stripes <Num Bytes per Stripes>
|
||||||
|
// 1. Calculate the number of stripes in this block group. <Num Stripes>
|
||||||
|
// 2. Calculate the last remaining length which does not make a stripe. <Last Stripe Length>
|
||||||
|
// 3. Total consumed space is the total of
|
||||||
|
// a. The total of the full cells of data blocks and parity blocks.
|
||||||
|
// b. The remaining of data block which does not make a stripe.
|
||||||
|
// c. The last parity block cells. These size should be same
|
||||||
|
// to the first cell in this stripe.
|
||||||
|
// So the total consumed space is the sum of
|
||||||
|
// a. <Cell Size> * (<Num Stripes> - 1) * <Total Block Num> = 0
|
||||||
|
// b. <Num Bytes> % <Num Bytes per Stripes> = 1
|
||||||
|
// c. <Last Stripe Length> * <Parity Block Num> = 1 * 3
|
||||||
|
assertEquals(4, inf.storagespaceConsumedWithStriped());
|
||||||
|
assertEquals(4, inf.storagespaceConsumed());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testMultipleBlockStripedConsumedSpace()
|
||||||
|
throws IOException, InterruptedException {
|
||||||
|
ECSchema defaultSchema = ECSchemaManager.getSystemDefaultSchema();
|
||||||
|
INodeFile inf = createStripedINodeFile();
|
||||||
|
inf.addStripedBlocksFeature();
|
||||||
|
Block blk1 = new Block(1);
|
||||||
|
BlockInfoStriped blockInfoStriped1
|
||||||
|
= new BlockInfoStriped(blk1,
|
||||||
|
(short)defaultSchema.getNumDataUnits(),
|
||||||
|
(short)defaultSchema.getNumParityUnits());
|
||||||
|
blockInfoStriped1.setNumBytes(1);
|
||||||
|
Block blk2 = new Block(2);
|
||||||
|
BlockInfoStriped blockInfoStriped2
|
||||||
|
= new BlockInfoStriped(blk2,
|
||||||
|
(short)defaultSchema.getNumDataUnits(),
|
||||||
|
(short)defaultSchema.getNumParityUnits());
|
||||||
|
blockInfoStriped2.setNumBytes(1);
|
||||||
|
inf.addBlock(blockInfoStriped1);
|
||||||
|
inf.addBlock(blockInfoStriped2);
|
||||||
|
// This is the double size of one block in above case.
|
||||||
|
assertEquals(4 * 2, inf.storagespaceConsumedWithStriped());
|
||||||
|
assertEquals(4 * 2, inf.storagespaceConsumed());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testBlockStripedFileSize()
|
||||||
|
throws IOException, InterruptedException {
|
||||||
|
ECSchema defaultSchema = ECSchemaManager.getSystemDefaultSchema();
|
||||||
|
INodeFile inf = createStripedINodeFile();
|
||||||
|
inf.addStripedBlocksFeature();
|
||||||
|
Block blk = new Block(1);
|
||||||
|
BlockInfoStriped blockInfoStriped
|
||||||
|
= new BlockInfoStriped(blk,
|
||||||
|
(short)defaultSchema.getNumDataUnits(),
|
||||||
|
(short)defaultSchema.getNumParityUnits());
|
||||||
|
blockInfoStriped.setNumBytes(100);
|
||||||
|
inf.addBlock(blockInfoStriped);
|
||||||
|
// Compute file size should return actual data
|
||||||
|
// size which is retained by this file.
|
||||||
|
assertEquals(100, inf.computeFileSize());
|
||||||
|
assertEquals(100, inf.computeFileSize(false, false));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testBlockStripedUCFileSize()
|
||||||
|
throws IOException, InterruptedException {
|
||||||
|
ECSchema defaultSchema = ECSchemaManager.getSystemDefaultSchema();
|
||||||
|
INodeFile inf = createStripedINodeFile();
|
||||||
|
inf.addStripedBlocksFeature();
|
||||||
|
Block blk = new Block(1);
|
||||||
|
BlockInfoStripedUnderConstruction bInfoStripedUC
|
||||||
|
= new BlockInfoStripedUnderConstruction(blk,
|
||||||
|
(short)defaultSchema.getNumDataUnits(),
|
||||||
|
(short)defaultSchema.getNumParityUnits());
|
||||||
|
bInfoStripedUC.setNumBytes(100);
|
||||||
|
inf.addBlock(bInfoStripedUC);
|
||||||
|
assertEquals(100, inf.computeFileSize());
|
||||||
|
assertEquals(0, inf.computeFileSize(false, false));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testBlockStripedComputeQuotaUsage()
|
||||||
|
throws IOException, InterruptedException {
|
||||||
|
ECSchema defaultSchema = ECSchemaManager.getSystemDefaultSchema();
|
||||||
|
INodeFile inf = createStripedINodeFile();
|
||||||
|
inf.addStripedBlocksFeature();
|
||||||
|
Block blk = new Block(1);
|
||||||
|
BlockInfoStriped blockInfoStriped
|
||||||
|
= new BlockInfoStriped(blk,
|
||||||
|
(short)defaultSchema.getNumDataUnits(),
|
||||||
|
(short)defaultSchema.getNumParityUnits());
|
||||||
|
blockInfoStriped.setNumBytes(100);
|
||||||
|
inf.addBlock(blockInfoStriped);
|
||||||
|
|
||||||
|
BlockStoragePolicySuite suite =
|
||||||
|
BlockStoragePolicySuite.createDefaultSuite();
|
||||||
|
QuotaCounts counts =
|
||||||
|
inf.computeQuotaUsageWithStriped(suite,
|
||||||
|
new QuotaCounts.Builder().build());
|
||||||
|
assertEquals(1, counts.getNameSpace());
|
||||||
|
// The total consumed space is the sum of
|
||||||
|
// a. <Cell Size> * (<Num Stripes> - 1) * <Total Block Num> = 0
|
||||||
|
// b. <Num Bytes> % <Num Bytes per Stripes> = 100
|
||||||
|
// c. <Last Stripe Length> * <Parity Block Num> = 100 * 3
|
||||||
|
assertEquals(400, counts.getStorageSpace());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testBlockStripedUCComputeQuotaUsage()
|
||||||
|
throws IOException, InterruptedException {
|
||||||
|
ECSchema defaultSchema = ECSchemaManager.getSystemDefaultSchema();
|
||||||
|
INodeFile inf = createStripedINodeFile();
|
||||||
|
inf.addStripedBlocksFeature();
|
||||||
|
Block blk = new Block(1);
|
||||||
|
BlockInfoStripedUnderConstruction bInfoStripedUC
|
||||||
|
= new BlockInfoStripedUnderConstruction(blk,
|
||||||
|
(short)defaultSchema.getNumDataUnits(),
|
||||||
|
(short)defaultSchema.getNumParityUnits());
|
||||||
|
bInfoStripedUC.setNumBytes(100);
|
||||||
|
inf.addBlock(bInfoStripedUC);
|
||||||
|
|
||||||
|
BlockStoragePolicySuite suite
|
||||||
|
= BlockStoragePolicySuite.createDefaultSuite();
|
||||||
|
QuotaCounts counts
|
||||||
|
= inf.computeQuotaUsageWithStriped(suite,
|
||||||
|
new QuotaCounts.Builder().build());
|
||||||
|
assertEquals(1024, inf.getPreferredBlockSize());
|
||||||
|
assertEquals(1, counts.getNameSpace());
|
||||||
|
// Consumed space in the case of BlockInfoStripedUC can be calculated
|
||||||
|
// by using preferred block size. This is 1024 and total block num
|
||||||
|
// is 9(= 3 + 6). Consumed storage space should be 1024 * 9 = 9216.
|
||||||
|
assertEquals(9216, counts.getStorageSpace());
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue