HDFS-10301. Interleaving processing of storages from repeated block reports causes false zombie storage detection, removes valid blocks. Contributed by Vinitha Gankidi.
This commit is contained in:
Vinitha Reddy Gankidi
Konstantin V Shvachko
parent
62800dc8b6
commit
9ab153d7ae
|
|
@ -63,6 +63,34 @@ public long[] getBlockListAsLongs() {
|
|||
public Iterator<BlockReportReplica> iterator() {
|
||||
return Collections.emptyIterator();
|
||||
}
|
||||
@Override
|
||||
public boolean isStorageReport() {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
// STORAGE_REPORT is used to report all storages in the DN
|
||||
public static final BlockListAsLongs STORAGE_REPORT = new BlockListAsLongs() {
|
||||
@Override
|
||||
public int getNumberOfBlocks() {
|
||||
return -1;
|
||||
}
|
||||
@Override
|
||||
public ByteString getBlocksBuffer() {
|
||||
return ByteString.EMPTY;
|
||||
}
|
||||
@Override
|
||||
public long[] getBlockListAsLongs() {
|
||||
return EMPTY_LONGS;
|
||||
}
|
||||
@Override
|
||||
public Iterator<BlockReportReplica> iterator() {
|
||||
return Collections.emptyIterator();
|
||||
}
|
||||
@Override
|
||||
public boolean isStorageReport() {
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
@ -252,6 +280,13 @@ public List<ByteString> getBlocksBuffers() {
|
|||
*/
|
||||
abstract public long[] getBlockListAsLongs();
|
||||
|
||||
/**
|
||||
* Return true for STORAGE_REPORT BlocksListsAsLongs.
|
||||
* Otherwise return false.
|
||||
* @return boolean
|
||||
*/
|
||||
abstract public boolean isStorageReport();
|
||||
|
||||
/**
|
||||
* Returns a singleton iterator over blocks in the block report. Do not
|
||||
* add the returned blocks to a collection.
|
||||
|
|
@ -391,6 +426,11 @@ public long[] getBlockListAsLongs() {
|
|||
return longs;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isStorageReport() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Iterator<BlockReportReplica> iterator() {
|
||||
return new Iterator<BlockReportReplica>() {
|
||||
|
|
@ -474,6 +514,11 @@ public long[] getBlockListAsLongs() {
|
|||
return longs;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isStorageReport() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Iterator<BlockReportReplica> iterator() {
|
||||
return new Iterator<BlockReportReplica>() {
|
||||
|
|
|
|||
|
|
@ -1889,8 +1889,8 @@ static class StatefulBlockInfo {
|
|||
*/
|
||||
public boolean processReport(final DatanodeID nodeID,
|
||||
final DatanodeStorage storage,
|
||||
final BlockListAsLongs newReport, BlockReportContext context,
|
||||
boolean lastStorageInRpc) throws IOException {
|
||||
final BlockListAsLongs newReport,
|
||||
BlockReportContext context) throws IOException {
|
||||
namesystem.writeLock();
|
||||
final long startTime = Time.monotonicNow(); //after acquiring write lock
|
||||
final long endTime;
|
||||
|
|
@ -1940,30 +1940,14 @@ public boolean processReport(final DatanodeID nodeID,
|
|||
|
||||
storageInfo.receivedBlockReport();
|
||||
if (context != null) {
|
||||
storageInfo.setLastBlockReportId(context.getReportId());
|
||||
if (lastStorageInRpc) {
|
||||
int rpcsSeen = node.updateBlockReportContext(context);
|
||||
if (rpcsSeen >= context.getTotalRpcs()) {
|
||||
long leaseId = blockReportLeaseManager.removeLease(node);
|
||||
BlockManagerFaultInjector.getInstance().
|
||||
removeBlockReportLease(node, leaseId);
|
||||
List<DatanodeStorageInfo> zombies = node.removeZombieStorages();
|
||||
if (zombies.isEmpty()) {
|
||||
LOG.debug("processReport 0x{}: no zombie storages found.",
|
||||
Long.toHexString(context.getReportId()));
|
||||
} else {
|
||||
for (DatanodeStorageInfo zombie : zombies) {
|
||||
removeZombieReplicas(context, zombie);
|
||||
}
|
||||
}
|
||||
node.clearBlockReportContext();
|
||||
} else {
|
||||
LOG.debug("processReport 0x{}: {} more RPCs remaining in this " +
|
||||
"report.", Long.toHexString(context.getReportId()),
|
||||
(context.getTotalRpcs() - rpcsSeen)
|
||||
);
|
||||
}
|
||||
if (context.getTotalRpcs() == context.getCurRpc() + 1) {
|
||||
long leaseId = this.getBlockReportLeaseManager().removeLease(node);
|
||||
BlockManagerFaultInjector.getInstance().
|
||||
removeBlockReportLease(node, leaseId);
|
||||
}
|
||||
LOG.debug("Processing RPC with index {} out of total {} RPCs in "
|
||||
+ "processReport 0x{}", context.getCurRpc(),
|
||||
context.getTotalRpcs(), Long.toHexString(context.getReportId()));
|
||||
}
|
||||
} finally {
|
||||
endTime = Time.monotonicNow();
|
||||
|
|
@ -1989,6 +1973,26 @@ public boolean processReport(final DatanodeID nodeID,
|
|||
return !node.hasStaleStorages();
|
||||
}
|
||||
|
||||
public void removeZombieStorages(DatanodeRegistration nodeReg,
|
||||
BlockReportContext context, Set<String> storageIDsInBlockReport)
|
||||
throws UnregisteredNodeException {
|
||||
namesystem.writeLock();
|
||||
DatanodeDescriptor node = this.getDatanodeManager().getDatanode(nodeReg);
|
||||
if (node != null) {
|
||||
List<DatanodeStorageInfo> zombies =
|
||||
node.removeZombieStorages(storageIDsInBlockReport);
|
||||
if (zombies.isEmpty()) {
|
||||
LOG.debug("processReport 0x{}: no zombie storages found.",
|
||||
Long.toHexString(context.getReportId()));
|
||||
} else {
|
||||
for (DatanodeStorageInfo zombie : zombies) {
|
||||
this.removeZombieReplicas(context, zombie);
|
||||
}
|
||||
}
|
||||
}
|
||||
namesystem.writeUnlock();
|
||||
}
|
||||
|
||||
private void removeZombieReplicas(BlockReportContext context,
|
||||
DatanodeStorageInfo zombie) {
|
||||
LOG.warn("processReport 0x{}: removing zombie storage {}, which no " +
|
||||
|
|
|
|||
|
|
@ -308,10 +308,10 @@ public synchronized boolean checkLease(DatanodeDescriptor dn,
|
|||
return false;
|
||||
}
|
||||
if (node.leaseId == 0) {
|
||||
LOG.warn("BR lease 0x{} is not valid for DN {}, because the DN " +
|
||||
LOG.warn("BR lease 0x{} is not found for DN {}, because the DN " +
|
||||
"is not in the pending set.",
|
||||
Long.toHexString(id), dn.getDatanodeUuid());
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
if (pruneIfExpired(monotonicNowMs, node)) {
|
||||
LOG.warn("BR lease 0x{} is not valid for DN {}, because the lease " +
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@
|
|||
package org.apache.hadoop.hdfs.server.blockmanagement;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.BitSet;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
|
|
@ -41,7 +40,6 @@
|
|||
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
|
||||
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
|
||||
import org.apache.hadoop.hdfs.server.namenode.CachedBlock;
|
||||
import org.apache.hadoop.hdfs.server.protocol.BlockReportContext;
|
||||
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
|
||||
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage.State;
|
||||
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
|
||||
|
|
@ -153,9 +151,6 @@ public Type getType() {
|
|||
public final DecommissioningStatus decommissioningStatus =
|
||||
new DecommissioningStatus();
|
||||
|
||||
private long curBlockReportId = 0;
|
||||
|
||||
private BitSet curBlockReportRpcsSeen = null;
|
||||
|
||||
private final Map<String, DatanodeStorageInfo> storageMap =
|
||||
new HashMap<>();
|
||||
|
|
@ -253,20 +248,6 @@ public DatanodeDescriptor(DatanodeID nodeID,
|
|||
updateHeartbeatState(StorageReport.EMPTY_ARRAY, 0L, 0L, 0, 0, null);
|
||||
}
|
||||
|
||||
public int updateBlockReportContext(BlockReportContext context) {
|
||||
if (curBlockReportId != context.getReportId()) {
|
||||
curBlockReportId = context.getReportId();
|
||||
curBlockReportRpcsSeen = new BitSet(context.getTotalRpcs());
|
||||
}
|
||||
curBlockReportRpcsSeen.set(context.getCurRpc());
|
||||
return curBlockReportRpcsSeen.cardinality();
|
||||
}
|
||||
|
||||
public void clearBlockReportContext() {
|
||||
curBlockReportId = 0;
|
||||
curBlockReportRpcsSeen = null;
|
||||
}
|
||||
|
||||
public CachedBlocksList getPendingCached() {
|
||||
return pendingCached;
|
||||
}
|
||||
|
|
@ -330,7 +311,8 @@ boolean hasStaleStorages() {
|
|||
}
|
||||
}
|
||||
|
||||
List<DatanodeStorageInfo> removeZombieStorages() {
|
||||
List<DatanodeStorageInfo>
|
||||
removeZombieStorages(Set<String> storageIDsInBlockReport) {
|
||||
List<DatanodeStorageInfo> zombies = null;
|
||||
synchronized (storageMap) {
|
||||
Iterator<Map.Entry<String, DatanodeStorageInfo>> iter =
|
||||
|
|
@ -338,18 +320,13 @@ List<DatanodeStorageInfo> removeZombieStorages() {
|
|||
while (iter.hasNext()) {
|
||||
Map.Entry<String, DatanodeStorageInfo> entry = iter.next();
|
||||
DatanodeStorageInfo storageInfo = entry.getValue();
|
||||
if (storageInfo.getLastBlockReportId() != curBlockReportId) {
|
||||
LOG.info("{} had lastBlockReportId 0x{} but curBlockReportId = 0x{}",
|
||||
storageInfo.getStorageID(),
|
||||
Long.toHexString(storageInfo.getLastBlockReportId()),
|
||||
Long.toHexString(curBlockReportId));
|
||||
if (!storageIDsInBlockReport.contains(storageInfo.getStorageID())) {
|
||||
iter.remove();
|
||||
if (zombies == null) {
|
||||
zombies = new LinkedList<>();
|
||||
}
|
||||
zombies.add(storageInfo);
|
||||
}
|
||||
storageInfo.setLastBlockReportId(0);
|
||||
}
|
||||
}
|
||||
return zombies == null ? EMPTY_STORAGE_INFO_LIST : zombies;
|
||||
|
|
|
|||
|
|
@ -122,9 +122,6 @@ public void remove() {
|
|||
private volatile BlockInfo blockList = null;
|
||||
private int numBlocks = 0;
|
||||
|
||||
// The ID of the last full block report which updated this storage.
|
||||
private long lastBlockReportId = 0;
|
||||
|
||||
/** The number of block reports received */
|
||||
private int blockReportCount = 0;
|
||||
|
||||
|
|
@ -189,14 +186,6 @@ public void setUtilizationForTesting(long capacity, long dfsUsed,
|
|||
this.blockPoolUsed = blockPoolUsed;
|
||||
}
|
||||
|
||||
long getLastBlockReportId() {
|
||||
return lastBlockReportId;
|
||||
}
|
||||
|
||||
void setLastBlockReportId(long lastBlockReportId) {
|
||||
this.lastBlockReportId = lastBlockReportId;
|
||||
}
|
||||
|
||||
State getState() {
|
||||
return this.state;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -367,11 +367,36 @@ List<DatanodeCommand> blockReport(long fullBrLeaseId) throws IOException {
|
|||
} else {
|
||||
// Send one block report per message.
|
||||
for (int r = 0; r < reports.length; r++) {
|
||||
StorageBlockReport singleReport[] = { reports[r] };
|
||||
DatanodeCommand cmd = bpNamenode.blockReport(
|
||||
bpRegistration, bpos.getBlockPoolId(), singleReport,
|
||||
new BlockReportContext(reports.length, r, reportId,
|
||||
fullBrLeaseId));
|
||||
StorageBlockReport[] singleReport = {reports[r]};
|
||||
DatanodeCommand cmd;
|
||||
if (r != reports.length - 1) {
|
||||
cmd = bpNamenode.blockReport(bpRegistration, bpos.getBlockPoolId(),
|
||||
singleReport, new BlockReportContext(reports.length, r,
|
||||
reportId, fullBrLeaseId));
|
||||
} else {
|
||||
StorageBlockReport[] lastSplitReport =
|
||||
new StorageBlockReport[perVolumeBlockLists.size()];
|
||||
// When block reports are split, the last RPC in the block report
|
||||
// has the information about all storages in the block report.
|
||||
// See HDFS-10301 for more details. To achieve this, the last RPC
|
||||
// has 'n' storage reports, where 'n' is the number of storages in
|
||||
// a DN. The actual block replicas are reported only for the
|
||||
// last/n-th storage.
|
||||
i = 0;
|
||||
for(Map.Entry<DatanodeStorage, BlockListAsLongs> kvPair :
|
||||
perVolumeBlockLists.entrySet()) {
|
||||
lastSplitReport[i++] = new StorageBlockReport(
|
||||
kvPair.getKey(), BlockListAsLongs.STORAGE_REPORT);
|
||||
if (i == r) {
|
||||
lastSplitReport[i] = reports[r];
|
||||
break;
|
||||
}
|
||||
}
|
||||
cmd = bpNamenode.blockReport(
|
||||
bpRegistration, bpos.getBlockPoolId(), lastSplitReport,
|
||||
new BlockReportContext(reports.length, r, reportId,
|
||||
fullBrLeaseId));
|
||||
}
|
||||
numReportsSent++;
|
||||
numRPCs++;
|
||||
if (cmd != null) {
|
||||
|
|
|
|||
|
|
@ -1415,24 +1415,36 @@ public DatanodeCommand blockReport(final DatanodeRegistration nodeReg,
|
|||
boolean noStaleStorages = false;
|
||||
for (int r = 0; r < reports.length; r++) {
|
||||
final BlockListAsLongs blocks = reports[r].getBlocks();
|
||||
//
|
||||
// BlockManager.processReport accumulates information of prior calls
|
||||
// for the same node and storage, so the value returned by the last
|
||||
// call of this loop is the final updated value for noStaleStorage.
|
||||
//
|
||||
final int index = r;
|
||||
noStaleStorages = bm.runBlockOp(new Callable<Boolean>() {
|
||||
@Override
|
||||
public Boolean call() throws IOException {
|
||||
return bm.processReport(nodeReg, reports[index].getStorage(),
|
||||
blocks, context, (index == reports.length - 1));
|
||||
}
|
||||
});
|
||||
metrics.incrStorageBlockReportOps();
|
||||
if (!blocks.isStorageReport()) {
|
||||
//
|
||||
// BlockManager.processReport accumulates information of prior calls
|
||||
// for the same node and storage, so the value returned by the last
|
||||
// call of this loop is the final updated value for noStaleStorage.
|
||||
//
|
||||
final int index = r;
|
||||
noStaleStorages = bm.runBlockOp(new Callable<Boolean>() {
|
||||
@Override
|
||||
public Boolean call()
|
||||
throws IOException {
|
||||
return bm.processReport(nodeReg, reports[index].getStorage(),
|
||||
blocks, context);
|
||||
}
|
||||
});
|
||||
metrics.incrStorageBlockReportOps();
|
||||
}
|
||||
}
|
||||
BlockManagerFaultInjector.getInstance().
|
||||
incomingBlockReportRpc(nodeReg, context);
|
||||
|
||||
if (nn.getFSImage().isUpgradeFinalized() &&
|
||||
context.getTotalRpcs() == context.getCurRpc() + 1) {
|
||||
Set<String> storageIDsInBlockReport = new HashSet<>();
|
||||
for (StorageBlockReport report : reports) {
|
||||
storageIDsInBlockReport.add(report.getStorage().getStorageID());
|
||||
}
|
||||
bm.removeZombieStorages(nodeReg, context, storageIDsInBlockReport);
|
||||
}
|
||||
|
||||
if (nn.getFSImage().isUpgradeFinalized() &&
|
||||
!namesystem.isRollingUpgrade() &&
|
||||
!nn.isStandbyState() &&
|
||||
|
|
|
|||
|
|
@ -710,12 +710,12 @@ public void testSafeModeIBR() throws Exception {
|
|||
reset(node);
|
||||
|
||||
bm.processReport(node, new DatanodeStorage(ds.getStorageID()),
|
||||
BlockListAsLongs.EMPTY, null, false);
|
||||
BlockListAsLongs.EMPTY, null);
|
||||
assertEquals(1, ds.getBlockReportCount());
|
||||
// send block report again, should NOT be processed
|
||||
reset(node);
|
||||
bm.processReport(node, new DatanodeStorage(ds.getStorageID()),
|
||||
BlockListAsLongs.EMPTY, null, false);
|
||||
BlockListAsLongs.EMPTY, null);
|
||||
assertEquals(1, ds.getBlockReportCount());
|
||||
|
||||
// re-register as if node restarted, should update existing node
|
||||
|
|
@ -726,7 +726,7 @@ public void testSafeModeIBR() throws Exception {
|
|||
// send block report, should be processed after restart
|
||||
reset(node);
|
||||
bm.processReport(node, new DatanodeStorage(ds.getStorageID()),
|
||||
BlockListAsLongs.EMPTY, null, false);
|
||||
BlockListAsLongs.EMPTY, null);
|
||||
// Reinitialize as registration with empty storage list pruned
|
||||
// node.storageMap.
|
||||
ds = node.getStorageInfos()[0];
|
||||
|
|
@ -755,7 +755,7 @@ public void testSafeModeIBRAfterIncremental() throws Exception {
|
|||
reset(node);
|
||||
doReturn(1).when(node).numBlocks();
|
||||
bm.processReport(node, new DatanodeStorage(ds.getStorageID()),
|
||||
BlockListAsLongs.EMPTY, null, false);
|
||||
BlockListAsLongs.EMPTY, null);
|
||||
assertEquals(1, ds.getBlockReportCount());
|
||||
}
|
||||
|
||||
|
|
@ -828,7 +828,7 @@ public void testSafeModeIBRBeforeFirstFullBR() throws Exception {
|
|||
// Make sure it's the first full report
|
||||
assertEquals(0, ds.getBlockReportCount());
|
||||
bm.processReport(node, new DatanodeStorage(ds.getStorageID()),
|
||||
builder.build(), null, false);
|
||||
builder.build(), null);
|
||||
assertEquals(1, ds.getBlockReportCount());
|
||||
|
||||
// verify the storage info is correct
|
||||
|
|
|
|||
|
|
@ -19,34 +19,40 @@
|
|||
package org.apache.hadoop.hdfs.server.blockmanagement;
|
||||
|
||||
import com.google.common.base.Supplier;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileUtil;
|
||||
import org.apache.hadoop.fs.FSDataOutputStream;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.hdfs.DFSConfigKeys;
|
||||
import org.apache.hadoop.hdfs.DFSTestUtil;
|
||||
import org.apache.hadoop.hdfs.DistributedFileSystem;
|
||||
import org.apache.hadoop.hdfs.HdfsConfiguration;
|
||||
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
||||
import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
|
||||
import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
|
||||
import org.apache.hadoop.hdfs.protocol.DatanodeID;
|
||||
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
|
||||
import org.apache.hadoop.hdfs.server.datanode.DataNode;
|
||||
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
|
||||
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
|
||||
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi.FsVolumeReferences;
|
||||
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
|
||||
import org.apache.hadoop.hdfs.server.protocol.BlockReportContext;
|
||||
import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand;
|
||||
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
|
||||
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
|
||||
import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport;
|
||||
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
|
||||
import org.apache.hadoop.test.GenericTestUtils;
|
||||
import org.apache.log4j.Level;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.BufferedOutputStream;
|
||||
import java.io.BufferedReader;
|
||||
import java.io.BufferedWriter;
|
||||
import java.io.File;
|
||||
|
|
@ -55,8 +61,6 @@
|
|||
import java.io.IOException;
|
||||
import java.io.OutputStreamWriter;
|
||||
import java.io.Writer;
|
||||
import java.util.Arrays;
|
||||
import java.util.EnumSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
|
|
@ -365,4 +369,67 @@ public Boolean get() {
|
|||
cluster.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
@Test(timeout=300000)
|
||||
public void testInterleavedFullBlockReports() throws Exception {
|
||||
Configuration conf = new HdfsConfiguration();
|
||||
conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY,
|
||||
36000000L);
|
||||
int numStoragesPerDatanode = 6;
|
||||
final MiniDFSCluster cluster = new MiniDFSCluster
|
||||
.Builder(conf).numDataNodes(1)
|
||||
.storagesPerDatanode(numStoragesPerDatanode)
|
||||
.build();
|
||||
try {
|
||||
LOG.info("waiting for cluster to become active...");
|
||||
cluster.waitActive();
|
||||
// Get the datanode registration and the block reports
|
||||
DataNode dn = cluster.getDataNodes().get(0);
|
||||
final String blockPoolId = cluster.getNamesystem().getBlockPoolId();
|
||||
LOG.info("Block pool id: " + blockPoolId);
|
||||
final DatanodeRegistration dnR = dn.getDNRegistrationForBP(blockPoolId);
|
||||
Map<DatanodeStorage, BlockListAsLongs> perVolumeBlockLists =
|
||||
dn.getFSDataset().getBlockReports(blockPoolId);
|
||||
final StorageBlockReport[] reports =
|
||||
new StorageBlockReport[perVolumeBlockLists.size()];
|
||||
int reportIndex = 0;
|
||||
for(Map.Entry<DatanodeStorage, BlockListAsLongs> kvPair :
|
||||
perVolumeBlockLists.entrySet()) {
|
||||
DatanodeStorage dnStorage = kvPair.getKey();
|
||||
BlockListAsLongs blockList = kvPair.getValue();
|
||||
reports[reportIndex++] =
|
||||
new StorageBlockReport(dnStorage, blockList);
|
||||
}
|
||||
// Get the list of storage ids associated with the datanode
|
||||
// before the test
|
||||
BlockManager bm =
|
||||
cluster.getNameNode().getNamesystem().getBlockManager();
|
||||
final DatanodeDescriptor dnDescriptor = bm.getDatanodeManager().
|
||||
getDatanode(cluster.getDataNodes().get(0).getDatanodeUuid());
|
||||
DatanodeStorageInfo[] storageInfos = dnDescriptor.getStorageInfos();
|
||||
// Send the full block report concurrently using
|
||||
// numThreads=numStoragesPerDatanode
|
||||
ExecutorService executorService = Executors.
|
||||
newFixedThreadPool(numStoragesPerDatanode);
|
||||
List<Future<DatanodeCommand>> futureList =
|
||||
new ArrayList<>(numStoragesPerDatanode);
|
||||
for (int i = 0; i < numStoragesPerDatanode; i++) {
|
||||
futureList.add(executorService.submit(new Callable<DatanodeCommand>() {
|
||||
@Override
|
||||
public DatanodeCommand call() throws IOException {
|
||||
return cluster.getNameNodeRpc().blockReport(dnR, blockPoolId,
|
||||
reports, new BlockReportContext(1, 0, System.nanoTime(), 0L));
|
||||
}
|
||||
}));
|
||||
}
|
||||
for (Future<DatanodeCommand> future: futureList) {
|
||||
future.get();
|
||||
}
|
||||
executorService.shutdown();
|
||||
// Verify that the storages match before and after the test
|
||||
Assert.assertArrayEquals(storageInfos, dnDescriptor.getStorageInfos());
|
||||
} finally {
|
||||
cluster.shutdown();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@
|
|||
import org.mockito.Mockito;
|
||||
|
||||
import static org.hamcrest.core.Is.is;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertThat;
|
||||
import static org.mockito.Matchers.*;
|
||||
import static org.mockito.Mockito.times;
|
||||
|
|
@ -88,6 +89,34 @@ private void createFile(String filenamePrefix, int blockCount)
|
|||
blockCount * BLOCK_SIZE, BLOCK_SIZE, REPL_FACTOR, seed);
|
||||
}
|
||||
|
||||
private void verifyCapturedArgumentsSplit(
|
||||
ArgumentCaptor<StorageBlockReport[]> captor,
|
||||
int expectedReportsPerCall,
|
||||
int expectedTotalBlockCount) {
|
||||
List<StorageBlockReport[]> listOfReports = captor.getAllValues();
|
||||
int numBlocksReported = 0;
|
||||
int storageIndex = 0;
|
||||
int listOfReportsSize = listOfReports.size();
|
||||
for (StorageBlockReport[] reports : listOfReports) {
|
||||
if (storageIndex < (listOfReportsSize - 1)) {
|
||||
assertThat(reports.length, is(expectedReportsPerCall));
|
||||
} else {
|
||||
assertThat(reports.length, is(listOfReportsSize));
|
||||
}
|
||||
for (StorageBlockReport report : reports) {
|
||||
BlockListAsLongs blockList = report.getBlocks();
|
||||
if (!blockList.isStorageReport()) {
|
||||
numBlocksReported += blockList.getNumberOfBlocks();
|
||||
} else {
|
||||
assertEquals(blockList.getNumberOfBlocks(), -1);
|
||||
}
|
||||
}
|
||||
storageIndex++;
|
||||
}
|
||||
|
||||
assert(numBlocksReported >= expectedTotalBlockCount);
|
||||
}
|
||||
|
||||
private void verifyCapturedArguments(
|
||||
ArgumentCaptor<StorageBlockReport[]> captor,
|
||||
int expectedReportsPerCall,
|
||||
|
|
@ -136,7 +165,7 @@ public void testAlwaysSplit() throws IOException, InterruptedException {
|
|||
anyString(),
|
||||
captor.capture(), Mockito.<BlockReportContext>anyObject());
|
||||
|
||||
verifyCapturedArguments(captor, 1, BLOCKS_IN_FILE);
|
||||
verifyCapturedArgumentsSplit(captor, 1, BLOCKS_IN_FILE);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -200,7 +229,7 @@ public void testCornerCaseAtThreshold() throws IOException, InterruptedException
|
|||
anyString(),
|
||||
captor.capture(), Mockito.<BlockReportContext>anyObject());
|
||||
|
||||
verifyCapturedArguments(captor, 1, BLOCKS_IN_FILE);
|
||||
verifyCapturedArgumentsSplit(captor, 1, BLOCKS_IN_FILE);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@
|
|||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
|
||||
import org.apache.hadoop.hdfs.server.protocol.BlockReportContext;
|
||||
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
|
||||
import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport;
|
||||
|
|
@ -35,13 +36,32 @@ public class TestNNHandlesBlockReportPerStorage extends BlockReportTestBase {
|
|||
@Override
|
||||
protected void sendBlockReports(DatanodeRegistration dnR, String poolId,
|
||||
StorageBlockReport[] reports) throws IOException {
|
||||
int i = 0;
|
||||
for (StorageBlockReport report : reports) {
|
||||
LOG.info("Sending block report for storage " + report.getStorage().getStorageID());
|
||||
StorageBlockReport[] singletonReport = { report };
|
||||
cluster.getNameNodeRpc().blockReport(dnR, poolId, singletonReport,
|
||||
new BlockReportContext(reports.length, i, System.nanoTime(), 0L));
|
||||
i++;
|
||||
for (int r = 0; r < reports.length; r++) {
|
||||
LOG.info("Sending block report for storage " +
|
||||
reports[r].getStorage().getStorageID());
|
||||
StorageBlockReport[] singletonReport = {reports[r]};
|
||||
if (r != reports.length - 1) {
|
||||
cluster.getNameNodeRpc().blockReport(dnR, poolId, singletonReport,
|
||||
new BlockReportContext(reports.length, r, System.nanoTime(),
|
||||
0L));
|
||||
} else {
|
||||
StorageBlockReport[] lastSplitReport =
|
||||
new StorageBlockReport[reports.length];
|
||||
// When block reports are split, send a dummy storage report for all
|
||||
// other storages in the blockreport along with the last storage report
|
||||
for (int i = 0; i <= r; i++) {
|
||||
if (i == r) {
|
||||
lastSplitReport[i] = reports[r];
|
||||
break;
|
||||
}
|
||||
lastSplitReport[i] =
|
||||
new StorageBlockReport(reports[i].getStorage(),
|
||||
BlockListAsLongs.STORAGE_REPORT);
|
||||
}
|
||||
cluster.getNameNodeRpc().blockReport(dnR, poolId, lastSplitReport,
|
||||
new BlockReportContext(reports.length, r, System.nanoTime(),
|
||||
0L));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue