From d611f42c37b660ba166815237b0dc02ebc6473cb Mon Sep 17 00:00:00 2001 From: Jing Zhao Date: Tue, 18 Mar 2014 00:10:22 +0000 Subject: [PATCH] Merge r1578669 from trunk. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1578676 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../org/apache/hadoop/fs/TestFileSystem.java | 3 ++- .../apache/hadoop/mapred/HadoopTestCase.java | 3 ++- .../mapred/TestCommandLineJobSubmission.java | 2 +- .../hadoop/mapred/TestJobSysDirWithDFS.java | 2 +- .../apache/hadoop/mapred/TestLazyOutput.java | 3 ++- .../mapred/TestMRCJCFileInputFormat.java | 11 ++++---- .../org/apache/hadoop/mapred/TestMerge.java | 3 ++- .../hadoop/mapred/TestMiniMRChildTask.java | 2 +- .../hadoop/mapred/TestMiniMRClasspath.java | 4 +-- .../hadoop/mapred/TestMiniMRDFSCaching.java | 2 +- .../TestMiniMRWithDFSWithDistinctUsers.java | 2 +- .../mapred/TestMultipleLevelCaching.java | 4 +-- .../mapred/TestReduceFetchFromPartialMem.java | 2 +- .../TestSpecialCharactersInOutputPath.java | 2 +- .../hadoop/mapred/join/TestDatamerge.java | 2 +- .../mapred/lib/TestDelegatingInputFormat.java | 7 ++--- .../apache/hadoop/mapred/pipes/TestPipes.java | 2 +- .../mapreduce/MiniHadoopClusterManager.java | 4 +-- .../mapreduce/TestMapReduceLazyOutput.java | 3 ++- .../lib/input/TestCombineFileInputFormat.java | 9 ++++--- .../lib/input/TestDelegatingInputFormat.java | 7 ++--- .../mapreduce/lib/join/TestJoinDatamerge.java | 2 +- .../lib/join/TestJoinProperties.java | 2 +- .../mapreduce/security/TestMRCredentials.java | 3 ++- .../security/ssl/TestEncryptedShuffle.java | 2 +- .../mapreduce/v2/TestMiniMRProxyUser.java | 3 ++- .../mapreduce/v2/TestNonExistentJob.java | 3 ++- .../contrib/utils/join/TestDataJoin.java | 2 +- .../hadoop/tools/TestGlobbedCopyListing.java | 2 +- .../apache/hadoop/tools/TestCopyFiles.java | 26 +++++++++---------- .../hadoop/streaming/TestDumpTypedBytes.java | 3 ++- .../apache/hadoop/streaming/TestFileArgs.java | 2 +- .../hadoop/streaming/TestLoadTypedBytes.java | 3 ++- .../streaming/TestMultipleArchiveFiles.java | 2 +- .../streaming/TestMultipleCachefiles.java | 2 +- .../apache/hadoop/streaming/TestSymLink.java | 2 +- 37 files changed, 79 insertions(+), 62 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 21438e3e68d..9cc0a05e382 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -165,6 +165,9 @@ Release 2.4.0 - UNRELEASED HDFS-6106. Reduce default for dfs.namenode.path.based.cache.refresh.interval.ms (cmccabe) + HDFS-6090. Use MiniDFSCluster.Builder instead of deprecated constructors. + (Akira AJISAKA via jing9) + OPTIMIZATIONS HDFS-5790. LeaseManager.findPath is very slow when many leases need recovery diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestFileSystem.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestFileSystem.java index f299cb67401..60c1ba6b847 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestFileSystem.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestFileSystem.java @@ -523,7 +523,8 @@ static void runTestCache(int port) throws Exception { Configuration conf = new Configuration(); MiniDFSCluster cluster = null; try { - cluster = new MiniDFSCluster(port, conf, 2, true, true, null, null); + cluster = new MiniDFSCluster.Builder(conf).nameNodePort(port) + .numDataNodes(2).build(); URI uri = cluster.getFileSystem().getUri(); LOG.info("uri=" + uri); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/HadoopTestCase.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/HadoopTestCase.java index c102e8f8626..3cd0668fe43 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/HadoopTestCase.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/HadoopTestCase.java @@ -146,7 +146,8 @@ protected void setUp() throws Exception { fileSystem = FileSystem.getLocal(new JobConf()); } else { - dfsCluster = new MiniDFSCluster(new JobConf(), dataNodes, true, null); + dfsCluster = new MiniDFSCluster.Builder(new JobConf()) + .numDataNodes(dataNodes).build(); fileSystem = dfsCluster.getFileSystem(); } if (localMR) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCommandLineJobSubmission.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCommandLineJobSubmission.java index 6e1a575e23f..69353871cf4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCommandLineJobSubmission.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCommandLineJobSubmission.java @@ -51,7 +51,7 @@ public void testJobShell() throws Exception { try { Configuration conf = new Configuration(); //start the mini mr and dfs cluster. - dfs = new MiniDFSCluster(conf, 2 , true, null); + dfs = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); fs = dfs.getFileSystem(); FSDataOutputStream stream = fs.create(testFile); stream.write("teststring".getBytes()); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobSysDirWithDFS.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobSysDirWithDFS.java index b32d5263c95..109c781c2b0 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobSysDirWithDFS.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobSysDirWithDFS.java @@ -125,7 +125,7 @@ public void testWithDFS() throws IOException { JobConf conf = new JobConf(); conf.set(JTConfig.JT_SYSTEM_DIR, "/tmp/custom/mapred/system"); - dfs = new MiniDFSCluster(conf, 4, true, null); + dfs = new MiniDFSCluster.Builder(conf).numDataNodes(4).build(); fileSys = dfs.getFileSystem(); mr = new MiniMRCluster(taskTrackers, fileSys.getUri().toString(), 1, null, null, conf); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestLazyOutput.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestLazyOutput.java index f8c94404f39..7412832d5c2 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestLazyOutput.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestLazyOutput.java @@ -141,7 +141,8 @@ public void testLazyOutput() throws Exception { Configuration conf = new Configuration(); // Start the mini-MR and mini-DFS clusters - dfs = new MiniDFSCluster(conf, NUM_HADOOP_SLAVES, true, null); + dfs = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_HADOOP_SLAVES) + .build(); fileSys = dfs.getFileSystem(); mr = new MiniMRCluster(NUM_HADOOP_SLAVES, fileSys.getUri().toString(), 1); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRCJCFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRCJCFileInputFormat.java index 8ec2ccfde92..fb9e8fcce3a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRCJCFileInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRCJCFileInputFormat.java @@ -44,11 +44,10 @@ public class TestMRCJCFileInputFormat extends TestCase { MiniDFSCluster dfs = null; private MiniDFSCluster newDFSCluster(JobConf conf) throws Exception { - return new MiniDFSCluster(conf, 4, true, - new String[]{"/rack0", "/rack0", - "/rack1", "/rack1"}, - new String[]{"host0", "host1", - "host2", "host3"}); + return new MiniDFSCluster.Builder(conf).numDataNodes(4) + .racks(new String[]{"/rack0", "/rack0", "/rack1", "/rack1"}) + .hosts(new String[]{"host0", "host1", "host2", "host3"}) + .build(); } public void testLocality() throws Exception { @@ -162,7 +161,7 @@ public void testMultiLevelInput() throws Exception { JobConf job = new JobConf(conf); job.setBoolean("dfs.replication.considerLoad", false); - dfs = new MiniDFSCluster(job, 1, true, rack1, hosts1); + dfs = new MiniDFSCluster.Builder(job).racks(rack1).hosts(hosts1).build(); dfs.waitActive(); String namenode = (dfs.getFileSystem()).getUri().getHost() + ":" + diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMerge.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMerge.java index 46a514f0f3e..e19ff589fa4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMerge.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMerge.java @@ -76,7 +76,8 @@ public void testMerge() throws Exception { try { Configuration conf = new Configuration(); // Start the mini-MR and mini-DFS clusters - dfsCluster = new MiniDFSCluster(conf, NUM_HADOOP_DATA_NODES, true, null); + dfsCluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(NUM_HADOOP_DATA_NODES).build(); fileSystem = dfsCluster.getFileSystem(); mrCluster = MiniMRClientClusterFactory.create(this.getClass(), NUM_HADOOP_DATA_NODES, conf); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java index f358874628d..ec447c28a79 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java @@ -324,7 +324,7 @@ public void reduce(WritableComparable key, Iterator values, @BeforeClass public static void setup() throws IOException { // create configuration, dfs, file system and mapred cluster - dfs = new MiniDFSCluster(conf, 1, true, null); + dfs = new MiniDFSCluster.Builder(conf).build(); fileSys = dfs.getFileSystem(); if (!(new File(MiniMRYarnCluster.APPJAR)).exists()) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClasspath.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClasspath.java index d5dcfac99b0..8984e374018 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClasspath.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClasspath.java @@ -169,7 +169,7 @@ public void testClassPath() throws IOException { final int jobTrackerPort = 60050; Configuration conf = new Configuration(); - dfs = new MiniDFSCluster(conf, 1, true, null); + dfs = new MiniDFSCluster.Builder(conf).build(); fileSys = dfs.getFileSystem(); namenode = fileSys.getUri().toString(); mr = new MiniMRCluster(taskTrackers, namenode, 3); @@ -201,7 +201,7 @@ public void testExternalWritable() final int taskTrackers = 4; Configuration conf = new Configuration(); - dfs = new MiniDFSCluster(conf, 1, true, null); + dfs = new MiniDFSCluster.Builder(conf).build(); fileSys = dfs.getFileSystem(); namenode = fileSys.getUri().toString(); mr = new MiniMRCluster(taskTrackers, namenode, 3); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRDFSCaching.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRDFSCaching.java index cda200e97e5..45879aff623 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRDFSCaching.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRDFSCaching.java @@ -38,7 +38,7 @@ public void testWithDFS() throws IOException { FileSystem fileSys = null; try { JobConf conf = new JobConf(); - dfs = new MiniDFSCluster(conf, 1, true, null); + dfs = new MiniDFSCluster.Builder(conf).build(); fileSys = dfs.getFileSystem(); mr = new MiniMRCluster(2, fileSys.getUri().toString(), 4); MRCaching.setupCache("/cachedir", fileSys); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java index 041f2dbfbff..221fd3c15c6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRWithDFSWithDistinctUsers.java @@ -75,7 +75,7 @@ public RunningJob run() throws IOException { @Before public void setUp() throws Exception { - dfs = new MiniDFSCluster(conf, 4, true, null); + dfs = new MiniDFSCluster.Builder(conf).numDataNodes(4).build(); fs = DFS_UGI.doAs(new PrivilegedExceptionAction() { public FileSystem run() throws IOException { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultipleLevelCaching.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultipleLevelCaching.java index cc9e88a2f09..294723a9c87 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultipleLevelCaching.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultipleLevelCaching.java @@ -92,8 +92,8 @@ private void testCachingAtLevel(int level) throws Exception { String rack2 = getRack(1, level); Configuration conf = new Configuration(); // Run a datanode on host1 under /a/b/c/..../d1/e1/f1 - dfs = new MiniDFSCluster(conf, 1, true, new String[] {rack1}, - new String[] {"host1.com"}); + dfs = new MiniDFSCluster.Builder(conf).racks(new String[] {rack1}) + .hosts(new String[] {"host1.com"}).build(); dfs.waitActive(); fileSys = dfs.getFileSystem(); if (!fileSys.mkdirs(inDir)) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestReduceFetchFromPartialMem.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestReduceFetchFromPartialMem.java index 4c33f9d4a33..3a1a275ab91 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestReduceFetchFromPartialMem.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestReduceFetchFromPartialMem.java @@ -57,7 +57,7 @@ public static Test suite() { TestSetup setup = new TestSetup(mySuite) { protected void setUp() throws Exception { Configuration conf = new Configuration(); - dfsCluster = new MiniDFSCluster(conf, 2, true, null); + dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); mrCluster = new MiniMRCluster(2, dfsCluster.getFileSystem().getUri().toString(), 1); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSpecialCharactersInOutputPath.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSpecialCharactersInOutputPath.java index 1b93377fd14..426686f9bb5 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSpecialCharactersInOutputPath.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestSpecialCharactersInOutputPath.java @@ -106,7 +106,7 @@ public void testJobWithDFS() throws IOException { final int taskTrackers = 4; final int jobTrackerPort = 60050; Configuration conf = new Configuration(); - dfs = new MiniDFSCluster(conf, 1, true, null); + dfs = new MiniDFSCluster.Builder(conf).build(); fileSys = dfs.getFileSystem(); namenode = fileSys.getUri().toString(); mr = new MiniMRCluster(taskTrackers, namenode, 2); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/join/TestDatamerge.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/join/TestDatamerge.java index e1cce0b56d6..15cea69dab2 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/join/TestDatamerge.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/join/TestDatamerge.java @@ -62,7 +62,7 @@ public static Test suite() { TestSetup setup = new TestSetup(new TestSuite(TestDatamerge.class)) { protected void setUp() throws Exception { Configuration conf = new Configuration(); - cluster = new MiniDFSCluster(conf, 2, true, null); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); } protected void tearDown() throws Exception { if (cluster != null) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestDelegatingInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestDelegatingInputFormat.java index 41dc5ebf8e9..8bd855433ea 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestDelegatingInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/TestDelegatingInputFormat.java @@ -39,9 +39,10 @@ public void testSplitting() throws Exception { JobConf conf = new JobConf(); MiniDFSCluster dfs = null; try { - dfs = new MiniDFSCluster(conf, 4, true, new String[] { "/rack0", - "/rack0", "/rack1", "/rack1" }, new String[] { "host0", "host1", - "host2", "host3" }); + dfs = new MiniDFSCluster.Builder(conf).numDataNodes(4) + .racks(new String[] { "/rack0", "/rack0", "/rack1", "/rack1" }) + .hosts(new String[] { "host0", "host1", "host2", "host3" }) + .build(); FileSystem fs = dfs.getFileSystem(); Path path = getPath("/foo/bar", fs); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipes.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipes.java index 46b6b59e8ca..dd7817d65b5 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipes.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipes.java @@ -79,7 +79,7 @@ public void testPipes() throws IOException { try { final int numSlaves = 2; Configuration conf = new Configuration(); - dfs = new MiniDFSCluster(conf, numSlaves, true, null); + dfs = new MiniDFSCluster.Builder(conf).numDataNodes(numSlaves).build(); mr = new MiniMRCluster(numSlaves, dfs.getFileSystem().getUri().toString(), 1); writeInputFile(dfs.getFileSystem(), inputPath); runProgram(mr, dfs, wordCountSimple, diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java index 35b5e30b4fc..2e8ba5e6270 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/MiniHadoopClusterManager.java @@ -151,8 +151,8 @@ private void sleepForever() { public void start() throws IOException, FileNotFoundException, URISyntaxException { if (!noDFS) { - dfs = new MiniDFSCluster(nnPort, conf, numDataNodes, true, true, - dfsOpts, null, null); + dfs = new MiniDFSCluster.Builder(conf).nameNodePort(nnPort) + .numDataNodes(numDataNodes).startupOption(dfsOpts).build(); LOG.info("Started MiniDFSCluster -- namenode on port " + dfs.getNameNodePort()); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMapReduceLazyOutput.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMapReduceLazyOutput.java index a1db55d05dd..1e4f4de9f93 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMapReduceLazyOutput.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMapReduceLazyOutput.java @@ -131,7 +131,8 @@ public void testLazyOutput() throws Exception { Configuration conf = new Configuration(); // Start the mini-MR and mini-DFS clusters - dfs = new MiniDFSCluster(conf, NUM_HADOOP_SLAVES, true, null); + dfs = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_HADOOP_SLAVES) + .build(); fileSys = dfs.getFileSystem(); mr = new MiniMRCluster(NUM_HADOOP_SLAVES, fileSys.getUri().toString(), 1); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java index 5730297b7e2..8b9a4102dc6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java @@ -313,7 +313,8 @@ public void testSplitPlacement() throws Exception { */ Configuration conf = new Configuration(); conf.setBoolean("dfs.replication.considerLoad", false); - dfs = new MiniDFSCluster(conf, 1, true, rack1, hosts1); + dfs = new MiniDFSCluster.Builder(conf).racks(rack1).hosts(hosts1) + .build(); dfs.waitActive(); fileSys = dfs.getFileSystem(); @@ -855,7 +856,8 @@ public void testSplitPlacementForCompressedFiles() throws Exception { */ Configuration conf = new Configuration(); conf.setBoolean("dfs.replication.considerLoad", false); - dfs = new MiniDFSCluster(conf, 1, true, rack1, hosts1); + dfs = new MiniDFSCluster.Builder(conf).racks(rack1).hosts(hosts1) + .build(); dfs.waitActive(); fileSys = dfs.getFileSystem(); @@ -1197,7 +1199,8 @@ public void testMissingBlocks() throws Exception { Configuration conf = new Configuration(); conf.set("fs.hdfs.impl", MissingBlockFileSystem.class.getName()); conf.setBoolean("dfs.replication.considerLoad", false); - dfs = new MiniDFSCluster(conf, 1, true, rack1, hosts1); + dfs = new MiniDFSCluster.Builder(conf).racks(rack1).hosts(hosts1) + .build(); dfs.waitActive(); namenode = (dfs.getFileSystem()).getUri().getHost() + ":" + diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestDelegatingInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestDelegatingInputFormat.java index 58054de4c5e..1428e472fac 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestDelegatingInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestDelegatingInputFormat.java @@ -37,9 +37,10 @@ public void testSplitting() throws Exception { Job job = Job.getInstance(); MiniDFSCluster dfs = null; try { - dfs = new MiniDFSCluster(job.getConfiguration(), 4, true, new String[] { "/rack0", - "/rack0", "/rack1", "/rack1" }, new String[] { "host0", "host1", - "host2", "host3" }); + dfs = new MiniDFSCluster.Builder(job.getConfiguration()).numDataNodes(4) + .racks(new String[] { "/rack0", "/rack0", "/rack1", "/rack1" }) + .hosts(new String[] { "host0", "host1", "host2", "host3" }) + .build(); FileSystem fs = dfs.getFileSystem(); Path path = getPath("/foo/bar", fs); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestJoinDatamerge.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestJoinDatamerge.java index 44bc43c3a19..d245bfd6cde 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestJoinDatamerge.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestJoinDatamerge.java @@ -45,7 +45,7 @@ public static Test suite() { TestSetup setup = new TestSetup(new TestSuite(TestJoinDatamerge.class)) { protected void setUp() throws Exception { Configuration conf = new Configuration(); - cluster = new MiniDFSCluster(conf, 2, true, null); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); } protected void tearDown() throws Exception { if (cluster != null) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestJoinProperties.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestJoinProperties.java index 858a02930ca..151bc875ad3 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestJoinProperties.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestJoinProperties.java @@ -50,7 +50,7 @@ public static Test suite() { TestSetup setup = new TestSetup(new TestSuite(TestJoinProperties.class)) { protected void setUp() throws Exception { Configuration conf = new Configuration(); - cluster = new MiniDFSCluster(conf, 2, true, null); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); base = cluster.getFileSystem().makeQualified(new Path("/nested")); src = generateSources(conf); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestMRCredentials.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestMRCredentials.java index a4969976672..72639e23dec 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestMRCredentials.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestMRCredentials.java @@ -58,7 +58,8 @@ public class TestMRCredentials { public static void setUp() throws Exception { System.setProperty("hadoop.log.dir", "logs"); Configuration conf = new Configuration(); - dfsCluster = new MiniDFSCluster(conf, numSlaves, true, null); + dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(numSlaves) + .build(); jConf = new JobConf(conf); FileSystem.setDefaultUri(conf, dfsCluster.getFileSystem().getUri().toString()); mrCluster = MiniMRClientClusterFactory.create(TestMRCredentials.class, 1, jConf); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java index 00c4f116455..d870d25b9af 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java @@ -92,7 +92,7 @@ private void startCluster(Configuration conf) throws Exception { YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)) + File.pathSeparator + classpathDir; conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH, cp); - dfsCluster = new MiniDFSCluster(conf, 1, true, null); + dfsCluster = new MiniDFSCluster.Builder(conf).build(); FileSystem fileSystem = dfsCluster.getFileSystem(); fileSystem.mkdirs(new Path("/tmp")); fileSystem.mkdirs(new Path("/user")); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMiniMRProxyUser.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMiniMRProxyUser.java index 41939cdec68..aa769f85974 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMiniMRProxyUser.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMiniMRProxyUser.java @@ -70,7 +70,8 @@ protected void setUp() throws Exception { UserGroupInformation.createUserForTesting("u1", userGroups); UserGroupInformation.createUserForTesting("u2", new String[]{"gg"}); - dfsCluster = new MiniDFSCluster(conf, dataNodes, true, null); + dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(dataNodes) + .build(); FileSystem fileSystem = dfsCluster.getFileSystem(); fileSystem.mkdirs(new Path("/tmp")); fileSystem.mkdirs(new Path("/user")); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestNonExistentJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestNonExistentJob.java index cdd1b550818..d4f5f843368 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestNonExistentJob.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestNonExistentJob.java @@ -57,7 +57,8 @@ protected void setUp() throws Exception { conf.set("dfs.permissions", "true"); conf.set("hadoop.security.authentication", "simple"); - dfsCluster = new MiniDFSCluster(conf, dataNodes, true, null); + dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(dataNodes) + .build(); FileSystem fileSystem = dfsCluster.getFileSystem(); fileSystem.mkdirs(new Path("/tmp")); fileSystem.mkdirs(new Path("/user")); diff --git a/hadoop-tools/hadoop-datajoin/src/test/java/org/apache/hadoop/contrib/utils/join/TestDataJoin.java b/hadoop-tools/hadoop-datajoin/src/test/java/org/apache/hadoop/contrib/utils/join/TestDataJoin.java index 37df5b002e6..dbb8ef0861e 100644 --- a/hadoop-tools/hadoop-datajoin/src/test/java/org/apache/hadoop/contrib/utils/join/TestDataJoin.java +++ b/hadoop-tools/hadoop-datajoin/src/test/java/org/apache/hadoop/contrib/utils/join/TestDataJoin.java @@ -43,7 +43,7 @@ public static Test suite() { TestSetup setup = new TestSetup(new TestSuite(TestDataJoin.class)) { protected void setUp() throws Exception { Configuration conf = new Configuration(); - cluster = new MiniDFSCluster(conf, 2, true, null); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); } protected void tearDown() throws Exception { if (cluster != null) { diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestGlobbedCopyListing.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestGlobbedCopyListing.java index 572a866fe6c..79099acf3f6 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestGlobbedCopyListing.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestGlobbedCopyListing.java @@ -49,7 +49,7 @@ public class TestGlobbedCopyListing { @BeforeClass public static void setup() throws Exception { - cluster = new MiniDFSCluster(new Configuration(), 1, true, null); + cluster = new MiniDFSCluster.Builder(new Configuration()).build(); createSourceData(); } diff --git a/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java b/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java index c2259e0eaee..ca7f36fca93 100644 --- a/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java +++ b/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java @@ -280,7 +280,7 @@ public void testCopyFromDfsToDfs() throws Exception { MiniDFSCluster cluster = null; try { Configuration conf = new Configuration(); - cluster = new MiniDFSCluster(conf, 2, true, null); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); final FileSystem hdfs = cluster.getFileSystem(); namenode = FileSystem.getDefaultUri(conf).toString(); if (namenode.startsWith("hdfs://")) { @@ -310,7 +310,7 @@ public void testEmptyDir() throws Exception { MiniDFSCluster cluster = null; try { Configuration conf = new Configuration(); - cluster = new MiniDFSCluster(conf, 2, true, null); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); final FileSystem hdfs = cluster.getFileSystem(); namenode = FileSystem.getDefaultUri(conf).toString(); if (namenode.startsWith("hdfs://")) { @@ -340,7 +340,7 @@ public void testCopyFromLocalToDfs() throws Exception { MiniDFSCluster cluster = null; try { Configuration conf = new Configuration(); - cluster = new MiniDFSCluster(conf, 1, true, null); + cluster = new MiniDFSCluster.Builder(conf).build(); final FileSystem hdfs = cluster.getFileSystem(); final String namenode = hdfs.getUri().toString(); if (namenode.startsWith("hdfs://")) { @@ -369,7 +369,7 @@ public void testCopyFromDfsToLocal() throws Exception { try { Configuration conf = new Configuration(); final FileSystem localfs = FileSystem.get(LOCAL_FS, conf); - cluster = new MiniDFSCluster(conf, 1, true, null); + cluster = new MiniDFSCluster.Builder(conf).build(); final FileSystem hdfs = cluster.getFileSystem(); final String namenode = FileSystem.getDefaultUri(conf).toString(); if (namenode.startsWith("hdfs://")) { @@ -396,7 +396,7 @@ public void testCopyDfsToDfsUpdateOverwrite() throws Exception { MiniDFSCluster cluster = null; try { Configuration conf = new Configuration(); - cluster = new MiniDFSCluster(conf, 2, true, null); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); final FileSystem hdfs = cluster.getFileSystem(); final String namenode = hdfs.getUri().toString(); if (namenode.startsWith("hdfs://")) { @@ -456,7 +456,7 @@ public void testCopyDfsToDfsUpdateWithSkipCRC() throws Exception { MiniDFSCluster cluster = null; try { Configuration conf = new Configuration(); - cluster = new MiniDFSCluster(conf, 2, true, null); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); final FileSystem hdfs = cluster.getFileSystem(); final String namenode = hdfs.getUri().toString(); @@ -614,7 +614,7 @@ public void testBasedir() throws Exception { MiniDFSCluster cluster = null; try { Configuration conf = new Configuration(); - cluster = new MiniDFSCluster(conf, 2, true, null); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); final FileSystem hdfs = cluster.getFileSystem(); namenode = FileSystem.getDefaultUri(conf).toString(); if (namenode.startsWith("hdfs://")) { @@ -639,7 +639,7 @@ public void testPreserveOption() throws Exception { Configuration conf = new Configuration(); MiniDFSCluster cluster = null; try { - cluster = new MiniDFSCluster(conf, 2, true, null); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); String nnUri = FileSystem.getDefaultUri(conf).toString(); FileSystem fs = FileSystem.get(URI.create(nnUri), conf); @@ -791,7 +791,7 @@ public void testLimits() throws Exception { Configuration conf = new Configuration(); MiniDFSCluster cluster = null; try { - cluster = new MiniDFSCluster(conf, 2, true, null); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); final String nnUri = FileSystem.getDefaultUri(conf).toString(); final FileSystem fs = FileSystem.get(URI.create(nnUri), conf); final DistCpV1 distcp = new DistCpV1(conf); @@ -899,7 +899,7 @@ public void testHftpAccessControl() throws Exception { //start cluster by DFS_UGI final Configuration dfsConf = new Configuration(); - cluster = new MiniDFSCluster(dfsConf, 2, true, null); + cluster = new MiniDFSCluster.Builder(dfsConf).numDataNodes(2).build(); cluster.waitActive(); final String httpAdd = dfsConf.get("dfs.http.address"); @@ -955,7 +955,7 @@ public void testDelete() throws Exception { conf.setInt("fs.trash.interval", 60); MiniDFSCluster cluster = null; try { - cluster = new MiniDFSCluster(conf, 2, true, null); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); final URI nnURI = FileSystem.getDefaultUri(conf); final String nnUri = nnURI.toString(); final FileSystem fs = FileSystem.get(URI.create(nnUri), conf); @@ -1027,7 +1027,7 @@ public void testDeleteLocal() throws Exception { try { Configuration conf = new Configuration(); final FileSystem localfs = FileSystem.get(LOCAL_FS, conf); - cluster = new MiniDFSCluster(conf, 1, true, null); + cluster = new MiniDFSCluster.Builder(conf).build(); final FileSystem hdfs = cluster.getFileSystem(); final String namenode = FileSystem.getDefaultUri(conf).toString(); if (namenode.startsWith("hdfs://")) { @@ -1060,7 +1060,7 @@ public void testGlobbing() throws Exception { MiniDFSCluster cluster = null; try { Configuration conf = new Configuration(); - cluster = new MiniDFSCluster(conf, 2, true, null); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); final FileSystem hdfs = cluster.getFileSystem(); namenode = FileSystem.getDefaultUri(conf).toString(); if (namenode.startsWith("hdfs://")) { diff --git a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestDumpTypedBytes.java b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestDumpTypedBytes.java index c290f50c577..bbfc02068e7 100644 --- a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestDumpTypedBytes.java +++ b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestDumpTypedBytes.java @@ -39,7 +39,8 @@ public class TestDumpTypedBytes { @Test public void testDumping() throws Exception { Configuration conf = new Configuration(); - MiniDFSCluster cluster = new MiniDFSCluster(conf, 2, true, null); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2) + .build(); FileSystem fs = cluster.getFileSystem(); PrintStream psBackup = System.out; ByteArrayOutputStream out = new ByteArrayOutputStream(); diff --git a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestFileArgs.java b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestFileArgs.java index e864e9d8555..fccd8d51e4b 100644 --- a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestFileArgs.java +++ b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestFileArgs.java @@ -54,7 +54,7 @@ public TestFileArgs() throws IOException { // Set up mini cluster conf = new Configuration(); - dfs = new MiniDFSCluster(conf, 1, true, null); + dfs = new MiniDFSCluster.Builder(conf).build(); fileSys = dfs.getFileSystem(); namenode = fileSys.getUri().getAuthority(); mr = new MiniMRCluster(1, namenode, 1); diff --git a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestLoadTypedBytes.java b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestLoadTypedBytes.java index df619f9ae87..0c004e7135f 100644 --- a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestLoadTypedBytes.java +++ b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestLoadTypedBytes.java @@ -39,7 +39,8 @@ public class TestLoadTypedBytes { @Test public void testLoading() throws Exception { Configuration conf = new Configuration(); - MiniDFSCluster cluster = new MiniDFSCluster(conf, 2, true, null); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2) + .build(); FileSystem fs = cluster.getFileSystem(); ByteArrayOutputStream out = new ByteArrayOutputStream(); diff --git a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleArchiveFiles.java b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleArchiveFiles.java index 47b70ef6fbd..8275c7e68cd 100644 --- a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleArchiveFiles.java +++ b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleArchiveFiles.java @@ -65,7 +65,7 @@ public TestMultipleArchiveFiles() throws Exception { input = "HADOOP"; expectedOutput = "HADOOP\t\nHADOOP\t\n"; conf = new Configuration(); - dfs = new MiniDFSCluster(conf, 1, true, null); + dfs = new MiniDFSCluster.Builder(conf).build(); fileSys = dfs.getFileSystem(); namenode = fileSys.getUri().getAuthority(); mr = new MiniMRCluster(1, namenode, 1); diff --git a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleCachefiles.java b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleCachefiles.java index ae8f57d231c..36565e3fc1b 100644 --- a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleCachefiles.java +++ b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleCachefiles.java @@ -69,7 +69,7 @@ public void testMultipleCachefiles() throws Exception MiniDFSCluster dfs = null; try{ Configuration conf = new Configuration(); - dfs = new MiniDFSCluster(conf, 1, true, null); + dfs = new MiniDFSCluster.Builder(conf).build(); FileSystem fileSys = dfs.getFileSystem(); String namenode = fileSys.getUri().toString(); diff --git a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestSymLink.java b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestSymLink.java index 730429d6daf..7994952578f 100644 --- a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestSymLink.java +++ b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestSymLink.java @@ -61,7 +61,7 @@ public void testSymLink() throws Exception MiniDFSCluster dfs = null; try { Configuration conf = new Configuration(); - dfs = new MiniDFSCluster(conf, 1, true, null); + dfs = new MiniDFSCluster.Builder(conf).build(); FileSystem fileSys = dfs.getFileSystem(); String namenode = fileSys.getUri().toString(); mr = new MiniMRCluster(1, namenode, 3);