MAPREDUCE-4110. Fix tests in TestMiniMRClasspath & TestMiniMRWithDFSWithDistinctUsers (Devaraj K via bobby)
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1310546 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
5af7077dcd
commit
b435e2202d
|
@ -213,7 +213,11 @@ Release 2.0.0 - UNRELEASED
|
|||
|
||||
MAPREDUCE-4111. Fix tests in org.apache.hadoop.mapred.TestJobName (Devaraj
|
||||
K via bobby)
|
||||
|
||||
|
||||
MAPREDUCE-4110. Fix tests in org.apache.hadoop.mapred.TestMiniMRClasspath &
|
||||
org.apache.hadoop.mapred.TestMiniMRWithDFSWithDistinctUsers (Devaraj K via
|
||||
bobby)
|
||||
|
||||
Release 0.23.3 - UNRELEASED
|
||||
|
||||
INCOMPATIBLE CHANGES
|
||||
|
|
|
@ -18,47 +18,43 @@
|
|||
|
||||
package org.apache.hadoop.mapred;
|
||||
|
||||
import java.io.*;
|
||||
import java.io.BufferedReader;
|
||||
import java.io.DataOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
import java.net.URI;
|
||||
|
||||
import junit.framework.TestCase;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.FileUtil;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
||||
import org.apache.hadoop.io.IntWritable;
|
||||
import org.apache.hadoop.io.Text;
|
||||
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
/**
|
||||
* A JUnit test to test Mini Map-Reduce Cluster with multiple directories
|
||||
* and check for correct classpath
|
||||
*/
|
||||
@Ignore
|
||||
public class TestMiniMRClasspath extends TestCase {
|
||||
public class TestMiniMRClasspath {
|
||||
|
||||
|
||||
static void configureWordCount(FileSystem fs,
|
||||
String jobTracker,
|
||||
JobConf conf,
|
||||
String input,
|
||||
int numMaps,
|
||||
int numReduces,
|
||||
Path inDir, Path outDir) throws IOException {
|
||||
static void configureWordCount(FileSystem fs, JobConf conf, String input,
|
||||
int numMaps, int numReduces, Path inDir, Path outDir) throws IOException {
|
||||
fs.delete(outDir, true);
|
||||
if (!fs.mkdirs(inDir)) {
|
||||
throw new IOException("Mkdirs failed to create " + inDir.toString());
|
||||
}
|
||||
{
|
||||
DataOutputStream file = fs.create(new Path(inDir, "part-0"));
|
||||
file.writeBytes(input);
|
||||
file.close();
|
||||
}
|
||||
DataOutputStream file = fs.create(new Path(inDir, "part-0"));
|
||||
file.writeBytes(input);
|
||||
file.close();
|
||||
FileSystem.setDefaultUri(conf, fs.getUri());
|
||||
conf.set(JTConfig.FRAMEWORK_NAME, JTConfig.CLASSIC_FRAMEWORK_NAME);
|
||||
conf.set(JTConfig.JT_IPC_ADDRESS, jobTracker);
|
||||
conf.set(JTConfig.FRAMEWORK_NAME, JTConfig.YARN_FRAMEWORK_NAME);
|
||||
conf.setJobName("wordcount");
|
||||
conf.setInputFormat(TextInputFormat.class);
|
||||
|
||||
|
@ -74,18 +70,17 @@ public class TestMiniMRClasspath extends TestCase {
|
|||
FileOutputFormat.setOutputPath(conf, outDir);
|
||||
conf.setNumMapTasks(numMaps);
|
||||
conf.setNumReduceTasks(numReduces);
|
||||
//pass a job.jar already included in the hadoop build
|
||||
conf.setJar("build/test/mapred/testjar/testjob.jar");
|
||||
//set the tests jar file
|
||||
conf.setJarByClass(TestMiniMRClasspath.class);
|
||||
}
|
||||
|
||||
static String launchWordCount(URI fileSys, String jobTracker, JobConf conf,
|
||||
String input, int numMaps, int numReduces)
|
||||
static String launchWordCount(URI fileSys, JobConf conf, String input,
|
||||
int numMaps, int numReduces)
|
||||
throws IOException {
|
||||
final Path inDir = new Path("/testing/wc/input");
|
||||
final Path outDir = new Path("/testing/wc/output");
|
||||
FileSystem fs = FileSystem.get(fileSys, conf);
|
||||
configureWordCount(fs, jobTracker, conf, input, numMaps, numReduces, inDir,
|
||||
outDir);
|
||||
configureWordCount(fs, conf, input, numMaps, numReduces, inDir, outDir);
|
||||
JobClient.runJob(conf);
|
||||
StringBuffer result = new StringBuffer();
|
||||
{
|
||||
|
@ -107,8 +102,8 @@ public class TestMiniMRClasspath extends TestCase {
|
|||
return result.toString();
|
||||
}
|
||||
|
||||
static String launchExternal(URI uri, String jobTracker, JobConf conf,
|
||||
String input, int numMaps, int numReduces)
|
||||
static String launchExternal(URI uri, JobConf conf, String input,
|
||||
int numMaps, int numReduces)
|
||||
throws IOException {
|
||||
|
||||
final Path inDir = new Path("/testing/ext/input");
|
||||
|
@ -124,8 +119,7 @@ public class TestMiniMRClasspath extends TestCase {
|
|||
file.close();
|
||||
}
|
||||
FileSystem.setDefaultUri(conf, uri);
|
||||
conf.set(JTConfig.FRAMEWORK_NAME, JTConfig.CLASSIC_FRAMEWORK_NAME);
|
||||
conf.set(JTConfig.JT_IPC_ADDRESS, jobTracker);
|
||||
conf.set(JTConfig.FRAMEWORK_NAME, JTConfig.YARN_FRAMEWORK_NAME);
|
||||
conf.setJobName("wordcount");
|
||||
conf.setInputFormat(TextInputFormat.class);
|
||||
|
||||
|
@ -142,8 +136,8 @@ public class TestMiniMRClasspath extends TestCase {
|
|||
conf.set("mapred.mapper.class", "testjar.ExternalMapperReducer");
|
||||
conf.set("mapred.reducer.class", "testjar.ExternalMapperReducer");
|
||||
|
||||
//pass a job.jar already included in the hadoop build
|
||||
conf.setJar("build/test/mapred/testjar/testjob.jar");
|
||||
// set the tests jar file
|
||||
conf.setJarByClass(TestMiniMRClasspath.class);
|
||||
JobClient.runJob(conf);
|
||||
StringBuffer result = new StringBuffer();
|
||||
Path[] fileList = FileUtil.stat2Paths(fs.listStatus(outDir,
|
||||
|
@ -164,6 +158,7 @@ public class TestMiniMRClasspath extends TestCase {
|
|||
return result.toString();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testClassPath() throws IOException {
|
||||
String namenode = null;
|
||||
MiniDFSCluster dfs = null;
|
||||
|
@ -180,13 +175,10 @@ public class TestMiniMRClasspath extends TestCase {
|
|||
mr = new MiniMRCluster(taskTrackers, namenode, 3);
|
||||
JobConf jobConf = new JobConf();
|
||||
String result;
|
||||
final String jobTrackerName = "localhost:" + mr.getJobTrackerPort();
|
||||
result = launchWordCount(fileSys.getUri(), jobTrackerName, jobConf,
|
||||
"The quick brown fox\nhas many silly\n" +
|
||||
"red fox sox\n",
|
||||
3, 1);
|
||||
assertEquals("The\t1\nbrown\t1\nfox\t2\nhas\t1\nmany\t1\n" +
|
||||
"quick\t1\nred\t1\nsilly\t1\nsox\t1\n", result);
|
||||
result = launchWordCount(fileSys.getUri(), jobConf,
|
||||
"The quick brown fox\nhas many silly\n" + "red fox sox\n", 3, 1);
|
||||
Assert.assertEquals("The\t1\nbrown\t1\nfox\t2\nhas\t1\nmany\t1\n"
|
||||
+ "quick\t1\nred\t1\nsilly\t1\nsox\t1\n", result);
|
||||
|
||||
} finally {
|
||||
if (dfs != null) { dfs.shutdown(); }
|
||||
|
@ -195,6 +187,7 @@ public class TestMiniMRClasspath extends TestCase {
|
|||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testExternalWritable()
|
||||
throws IOException {
|
||||
|
||||
|
@ -214,12 +207,10 @@ public class TestMiniMRClasspath extends TestCase {
|
|||
mr = new MiniMRCluster(taskTrackers, namenode, 3);
|
||||
JobConf jobConf = new JobConf();
|
||||
String result;
|
||||
final String jobTrackerName = "localhost:" + mr.getJobTrackerPort();
|
||||
|
||||
result = launchExternal(fileSys.getUri(), jobTrackerName, jobConf,
|
||||
"Dennis was here!\nDennis again!",
|
||||
3, 1);
|
||||
assertEquals("Dennis again!\t1\nDennis was here!\t1\n", result);
|
||||
result = launchExternal(fileSys.getUri(), jobConf,
|
||||
"Dennis was here!\nDennis again!", 3, 1);
|
||||
Assert.assertEquals("Dennis again!\t1\nDennis was here!\t1\n", result);
|
||||
|
||||
}
|
||||
finally {
|
||||
|
|
|
@ -17,26 +17,26 @@
|
|||
*/
|
||||
package org.apache.hadoop.mapred;
|
||||
|
||||
import java.io.*;
|
||||
import java.io.IOException;
|
||||
import java.security.PrivilegedExceptionAction;
|
||||
|
||||
import junit.framework.TestCase;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.fs.permission.FsPermission;
|
||||
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
||||
import org.apache.hadoop.mapreduce.MRJobConfig;
|
||||
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
|
||||
import org.apache.hadoop.security.*;
|
||||
import org.junit.Ignore;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.junit.After;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
/**
|
||||
* A JUnit test to test Mini Map-Reduce Cluster with Mini-DFS.
|
||||
*/
|
||||
@Ignore
|
||||
public class TestMiniMRWithDFSWithDistinctUsers extends TestCase {
|
||||
public class TestMiniMRWithDFSWithDistinctUsers {
|
||||
static final UserGroupInformation DFS_UGI = createUGI("dfs", true);
|
||||
static final UserGroupInformation ALICE_UGI = createUGI("alice", false);
|
||||
static final UserGroupInformation BOB_UGI = createUGI("bob", false);
|
||||
|
@ -45,7 +45,6 @@ public class TestMiniMRWithDFSWithDistinctUsers extends TestCase {
|
|||
MiniDFSCluster dfs = null;
|
||||
FileSystem fs = null;
|
||||
Configuration conf = new Configuration();
|
||||
String jobTrackerName;
|
||||
|
||||
static UserGroupInformation createUGI(String name, boolean issuper) {
|
||||
String group = issuper? "supergroup": name;
|
||||
|
@ -71,9 +70,10 @@ public class TestMiniMRWithDFSWithDistinctUsers extends TestCase {
|
|||
});
|
||||
|
||||
rj.waitForCompletion();
|
||||
assertEquals("SUCCEEDED", JobStatus.getJobRunState(rj.getJobState()));
|
||||
Assert.assertEquals("SUCCEEDED", JobStatus.getJobRunState(rj.getJobState()));
|
||||
}
|
||||
|
||||
@Before
|
||||
public void setUp() throws Exception {
|
||||
dfs = new MiniDFSCluster(conf, 4, true, null);
|
||||
|
||||
|
@ -96,29 +96,30 @@ public class TestMiniMRWithDFSWithDistinctUsers extends TestCase {
|
|||
|
||||
mr = new MiniMRCluster(0, 0, 4, dfs.getFileSystem().getUri().toString(),
|
||||
1, null, null, MR_UGI, mrConf);
|
||||
jobTrackerName = "localhost:" + mr.getJobTrackerPort();
|
||||
}
|
||||
|
||||
@After
|
||||
public void tearDown() throws Exception {
|
||||
if (mr != null) { mr.shutdown();}
|
||||
if (dfs != null) { dfs.shutdown(); }
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDistinctUsers() throws Exception {
|
||||
JobConf job1 = mr.createJobConf();
|
||||
String input = "The quick brown fox\nhas many silly\n"
|
||||
+ "red fox sox\n";
|
||||
Path inDir = new Path("/testing/distinct/input");
|
||||
Path outDir = new Path("/user/alice/output");
|
||||
TestMiniMRClasspath.configureWordCount(fs, jobTrackerName, job1,
|
||||
input, 2, 1, inDir, outDir);
|
||||
TestMiniMRClasspath
|
||||
.configureWordCount(fs, job1, input, 2, 1, inDir, outDir);
|
||||
runJobAsUser(job1, ALICE_UGI);
|
||||
|
||||
JobConf job2 = mr.createJobConf();
|
||||
Path inDir2 = new Path("/testing/distinct/input2");
|
||||
Path outDir2 = new Path("/user/bob/output2");
|
||||
TestMiniMRClasspath.configureWordCount(fs, jobTrackerName, job2,
|
||||
input, 2, 1, inDir2, outDir2);
|
||||
TestMiniMRClasspath.configureWordCount(fs, job2, input, 2, 1, inDir2,
|
||||
outDir2);
|
||||
runJobAsUser(job2, BOB_UGI);
|
||||
}
|
||||
|
||||
|
@ -127,6 +128,7 @@ public class TestMiniMRWithDFSWithDistinctUsers extends TestCase {
|
|||
* task makes lots of spills (more than fit in the spill index cache)
|
||||
* that it will succeed.
|
||||
*/
|
||||
@Test
|
||||
public void testMultipleSpills() throws Exception {
|
||||
JobConf job1 = mr.createJobConf();
|
||||
|
||||
|
@ -141,8 +143,8 @@ public class TestMiniMRWithDFSWithDistinctUsers extends TestCase {
|
|||
+ "red fox sox\n";
|
||||
Path inDir = new Path("/testing/distinct/input");
|
||||
Path outDir = new Path("/user/alice/output");
|
||||
TestMiniMRClasspath.configureWordCount(fs, jobTrackerName, job1,
|
||||
input, 2, 1, inDir, outDir);
|
||||
TestMiniMRClasspath
|
||||
.configureWordCount(fs, job1, input, 2, 1, inDir, outDir);
|
||||
runJobAsUser(job1, ALICE_UGI);
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue