HDFS-10287. MiniDFSCluster should implement AutoCloseable. Contributed by Andras Bokor.
(cherry picked from commit fcde6940e0
)
This commit is contained in:
parent
419d6ce19c
commit
1222889f1a
|
@ -136,7 +136,7 @@ import com.google.common.collect.Sets;
|
||||||
*/
|
*/
|
||||||
@InterfaceAudience.LimitedPrivate({"HBase", "HDFS", "Hive", "MapReduce", "Pig"})
|
@InterfaceAudience.LimitedPrivate({"HBase", "HDFS", "Hive", "MapReduce", "Pig"})
|
||||||
@InterfaceStability.Unstable
|
@InterfaceStability.Unstable
|
||||||
public class MiniDFSCluster {
|
public class MiniDFSCluster implements AutoCloseable {
|
||||||
|
|
||||||
private static final String NAMESERVICE_ID_PREFIX = "nameserviceId";
|
private static final String NAMESERVICE_ID_PREFIX = "nameserviceId";
|
||||||
private static final Log LOG = LogFactory.getLog(MiniDFSCluster.class);
|
private static final Log LOG = LogFactory.getLog(MiniDFSCluster.class);
|
||||||
|
@ -2971,4 +2971,9 @@ public class MiniDFSCluster {
|
||||||
writer.close();
|
writer.close();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void close() {
|
||||||
|
shutdown();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -64,22 +64,17 @@ public class TestMiniDFSCluster {
|
||||||
public void testClusterWithoutSystemProperties() throws Throwable {
|
public void testClusterWithoutSystemProperties() throws Throwable {
|
||||||
String oldPrp = System.getProperty(MiniDFSCluster.PROP_TEST_BUILD_DATA);
|
String oldPrp = System.getProperty(MiniDFSCluster.PROP_TEST_BUILD_DATA);
|
||||||
System.clearProperty(MiniDFSCluster.PROP_TEST_BUILD_DATA);
|
System.clearProperty(MiniDFSCluster.PROP_TEST_BUILD_DATA);
|
||||||
MiniDFSCluster cluster = null;
|
|
||||||
try {
|
|
||||||
Configuration conf = new HdfsConfiguration();
|
Configuration conf = new HdfsConfiguration();
|
||||||
File testDataCluster1 = new File(testDataPath, CLUSTER_1);
|
File testDataCluster1 = new File(testDataPath, CLUSTER_1);
|
||||||
String c1Path = testDataCluster1.getAbsolutePath();
|
String c1Path = testDataCluster1.getAbsolutePath();
|
||||||
conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, c1Path);
|
conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, c1Path);
|
||||||
cluster = new MiniDFSCluster.Builder(conf).build();
|
try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build()){
|
||||||
assertEquals(new File(c1Path + "/data"),
|
assertEquals(new File(c1Path + "/data"),
|
||||||
new File(cluster.getDataDirectory()));
|
new File(cluster.getDataDirectory()));
|
||||||
} finally {
|
} finally {
|
||||||
if (oldPrp != null) {
|
if (oldPrp != null) {
|
||||||
System.setProperty(MiniDFSCluster.PROP_TEST_BUILD_DATA, oldPrp);
|
System.setProperty(MiniDFSCluster.PROP_TEST_BUILD_DATA, oldPrp);
|
||||||
}
|
}
|
||||||
if (cluster != null) {
|
|
||||||
cluster.shutdown();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -110,15 +105,12 @@ public class TestMiniDFSCluster {
|
||||||
File testDataCluster5 = new File(testDataPath, CLUSTER_5);
|
File testDataCluster5 = new File(testDataPath, CLUSTER_5);
|
||||||
String c5Path = testDataCluster5.getAbsolutePath();
|
String c5Path = testDataCluster5.getAbsolutePath();
|
||||||
conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, c5Path);
|
conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, c5Path);
|
||||||
MiniDFSCluster cluster5 = new MiniDFSCluster.Builder(conf)
|
try (MiniDFSCluster cluster5 = new MiniDFSCluster.Builder(conf)
|
||||||
.numDataNodes(1)
|
.numDataNodes(1)
|
||||||
.checkDataNodeHostConfig(true)
|
.checkDataNodeHostConfig(true)
|
||||||
.build();
|
.build()) {
|
||||||
try {
|
|
||||||
assertEquals("DataNode hostname config not respected", "MYHOST",
|
assertEquals("DataNode hostname config not respected", "MYHOST",
|
||||||
cluster5.getDataNodes().get(0).getDatanodeId().getHostName());
|
cluster5.getDataNodes().get(0).getDatanodeId().getHostName());
|
||||||
} finally {
|
|
||||||
MiniDFSCluster.shutdownCluster(cluster5);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -128,9 +120,8 @@ public class TestMiniDFSCluster {
|
||||||
StorageType[][] storageType = new StorageType[][] {
|
StorageType[][] storageType = new StorageType[][] {
|
||||||
{StorageType.DISK, StorageType.ARCHIVE}, {StorageType.DISK},
|
{StorageType.DISK, StorageType.ARCHIVE}, {StorageType.DISK},
|
||||||
{StorageType.ARCHIVE}};
|
{StorageType.ARCHIVE}};
|
||||||
final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
|
try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
|
||||||
.numDataNodes(3).storageTypes(storageType).build();
|
.numDataNodes(3).storageTypes(storageType).build()) {
|
||||||
try {
|
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
ArrayList<DataNode> dataNodes = cluster.getDataNodes();
|
ArrayList<DataNode> dataNodes = cluster.getDataNodes();
|
||||||
// Check the number of directory in DN's
|
// Check the number of directory in DN's
|
||||||
|
@ -138,17 +129,14 @@ public class TestMiniDFSCluster {
|
||||||
assertEquals(DataNode.getStorageLocations(dataNodes.get(i).getConf())
|
assertEquals(DataNode.getStorageLocations(dataNodes.get(i).getConf())
|
||||||
.size(), storageType[i].length);
|
.size(), storageType[i].length);
|
||||||
}
|
}
|
||||||
} finally {
|
|
||||||
MiniDFSCluster.shutdownCluster(cluster);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testClusterNoStorageTypeSetForDatanodes() throws IOException {
|
public void testClusterNoStorageTypeSetForDatanodes() throws IOException {
|
||||||
final Configuration conf = new HdfsConfiguration();
|
final Configuration conf = new HdfsConfiguration();
|
||||||
final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
|
try (final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
|
||||||
.numDataNodes(3).build();
|
.numDataNodes(3).build()) {
|
||||||
try {
|
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
ArrayList<DataNode> dataNodes = cluster.getDataNodes();
|
ArrayList<DataNode> dataNodes = cluster.getDataNodes();
|
||||||
// Check the number of directory in DN's
|
// Check the number of directory in DN's
|
||||||
|
@ -156,20 +144,17 @@ public class TestMiniDFSCluster {
|
||||||
assertEquals(DataNode.getStorageLocations(datanode.getConf()).size(),
|
assertEquals(DataNode.getStorageLocations(datanode.getConf()).size(),
|
||||||
2);
|
2);
|
||||||
}
|
}
|
||||||
} finally {
|
|
||||||
MiniDFSCluster.shutdownCluster(cluster);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testSetUpFederatedCluster() throws Exception {
|
public void testSetUpFederatedCluster() throws Exception {
|
||||||
Configuration conf = new Configuration();
|
Configuration conf = new Configuration();
|
||||||
MiniDFSCluster cluster =
|
|
||||||
new MiniDFSCluster.Builder(conf).nnTopology(
|
try (MiniDFSCluster cluster =
|
||||||
MiniDFSNNTopology.simpleHAFederatedTopology(2))
|
new MiniDFSCluster.Builder(conf)
|
||||||
.numDataNodes(2)
|
.nnTopology(MiniDFSNNTopology.simpleHAFederatedTopology(2))
|
||||||
.build();
|
.numDataNodes(2).build()) {
|
||||||
try {
|
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
cluster.transitionToActive(1);
|
cluster.transitionToActive(1);
|
||||||
cluster.transitionToActive(3);
|
cluster.transitionToActive(3);
|
||||||
|
@ -201,8 +186,6 @@ public class TestMiniDFSCluster {
|
||||||
DFSUtil.addKeySuffixes(
|
DFSUtil.addKeySuffixes(
|
||||||
DFS_NAMENODE_HTTP_ADDRESS_KEY, "ns1", "nn1")));
|
DFS_NAMENODE_HTTP_ADDRESS_KEY, "ns1", "nn1")));
|
||||||
}
|
}
|
||||||
} finally {
|
|
||||||
MiniDFSCluster.shutdownCluster(cluster);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue