Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-9619. SimulatedFSDataset sometimes can not find blockpool for the correct namenode (Contributed by Wei-Chiu Chuang)

This commit is contained in:
Vinayakumar B 2016-01-07 14:20:18 +05:30
parent 34cd7cd765
commit 6702e7d66a
3 changed files with 52 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -2567,6 +2567,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9605. Add links to failed volumes to explorer.html in HDFS Web UI.
(Archana T via wheat9)
HDFS-9619. SimulatedFSDataset sometimes can not find blockpool for the
correct namenode (Wei-Chiu Chuang via vinayakumarb)
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
View File

@ -29,6 +29,7 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import javax.management.NotCompliantMBeanException;
import javax.management.ObjectName;
@ -521,7 +522,7 @@ public FsDatasetSpi getDataset() {
}
private final Map<String, Map<Block, BInfo>> blockMap
= new HashMap<String, Map<Block,BInfo>>();
= new ConcurrentHashMap<String, Map<Block,BInfo>>();
private final SimulatedStorage storage;
private final SimulatedVolume volume;
private final String datanodeUuid;

47
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java
View File

@ -27,6 +27,7 @@
import java.io.InputStream;
import java.io.OutputStream;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.HdfsConfiguration;
@ -38,6 +39,7 @@
import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaOutputStreams;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetFactory;
import org.apache.hadoop.util.DataChecksum;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@ -338,4 +340,49 @@ private SimulatedFSDataset getSimulatedFSDataset() {
fsdataset.addBlockPool(bpid, conf);
return fsdataset;
}
@Test
public void testConcurrentAddBlockPool() throws InterruptedException,
IOException {
final String[] bpids = {"BP-TEST1-", "BP-TEST2-"};
final SimulatedFSDataset fsdataset = new SimulatedFSDataset(null, conf);
class AddBlockPoolThread extends Thread {
private int id;
private IOException ioe;
public AddBlockPoolThread(int id) {
super();
this.id = id;
}
public void test() throws InterruptedException, IOException {
this.join();
if (ioe != null) {
throw ioe;
}
}
public void run() {
for (int i=0; i < 10000; i++) {
// add different block pools concurrently
String newbpid = bpids[id] + i;
fsdataset.addBlockPool(newbpid, conf);
// and then add a block into the pool
ExtendedBlock block = new ExtendedBlock(newbpid,1);
try {
// it will throw an exception if the block pool is not found
fsdataset.createTemporary(StorageType.DEFAULT, block);
} catch (IOException ioe) {
// JUnit does not capture exception in non-main thread,
// so cache it and then let main thread throw later.
this.ioe = ioe;
}
assert(fsdataset.getReplicaString(newbpid,1) != "null");
}
}
};
AddBlockPoolThread t1 = new AddBlockPoolThread(0);
AddBlockPoolThread t2 = new AddBlockPoolThread(1);
t1.start();
t2.start();
t1.test();
t2.test();
}
}