From 16a44b6fa5ef94bbfb7e12fe846805070ccd60c4 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Tue, 14 Feb 2012 21:39:59 +0000 Subject: [PATCH 1/6] MAPREDUCE-3854. Fixed and reenabled tests related to MR child JVM's environmental variables in TestMiniMRChildTask. (Tom White via vinodkv) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1244223 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../hadoop/mapred/TestMiniMRChildTask.java | 17 +++++++++-------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index e0504b90481..74dacd22633 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -94,6 +94,9 @@ Release 0.23.2 - UNRELEASED IMPROVEMENTS + MAPREDUCE-3854. Fixed and reenabled tests related to MR child JVM's + environmental variables in TestMiniMRChildTask. (Tom White via vinodkv) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java index 49c899cf1d5..2b5ffd4f0db 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java @@ -219,8 +219,8 @@ public class TestMiniMRChildTask { // check if X=$X:/abc works for LD_LIBRARY_PATH checkEnv("LD_LIBRARY_PATH", "/tmp", "append"); - // check if X=/tmp works for an already existing parameter - checkEnv("HOME", "/tmp", "noappend"); + // check if X=y works for an already existing parameter + checkEnv("LANG", "en_us_8859_1", "noappend"); // check if X=/tmp for a new env variable checkEnv("MY_PATH", "/tmp", "noappend"); // check if X=$X:/tmp works for a new env var and results into :/tmp @@ -269,8 +269,8 @@ public class TestMiniMRChildTask { // check if X=$X:/abc works for LD_LIBRARY_PATH checkEnv("LD_LIBRARY_PATH", "/tmp", "append"); - // check if X=/tmp works for an already existing parameter - checkEnv("HOME", "/tmp", "noappend"); + // check if X=y works for an already existing parameter + checkEnv("LANG", "en_us_8859_1", "noappend"); // check if X=/tmp for a new env variable checkEnv("MY_PATH", "/tmp", "noappend"); // check if X=$X:/tmp works for a new env var and results into :/tmp @@ -369,7 +369,7 @@ public class TestMiniMRChildTask { * - x=y (x can be a already existing env variable or a new variable) * - x=$x:y (replace $x with the current value of x) */ - + @Test public void testTaskEnv(){ try { JobConf conf = new JobConf(mr.getConfig()); @@ -392,6 +392,7 @@ public class TestMiniMRChildTask { * - x=y (x can be a already existing env variable or a new variable) * - x=$x:y (replace $x with the current value of x) */ + @Test public void testTaskOldEnv(){ try { JobConf conf = new JobConf(mr.getConfig()); @@ -415,7 +416,7 @@ public class TestMiniMRChildTask { EnvCheckMapper.class, EnvCheckReducer.class); // test // - new SET of new var (MY_PATH) - // - set of old var (HOME) + // - set of old var (LANG) // - append to an old var from modified env (LD_LIBRARY_PATH) // - append to an old var from tt's env (PATH) // - append to a new var (NEW_PATH) @@ -432,10 +433,10 @@ public class TestMiniMRChildTask { mapTaskJavaOpts = reduceTaskJavaOpts = TASK_OPTS_VAL; } conf.set(mapTaskEnvKey, - "MY_PATH=/tmp,HOME=/tmp,LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/tmp," + + "MY_PATH=/tmp,LANG=en_us_8859_1,LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/tmp," + "PATH=$PATH:/tmp,NEW_PATH=$NEW_PATH:/tmp"); conf.set(reduceTaskEnvKey, - "MY_PATH=/tmp,HOME=/tmp,LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/tmp," + + "MY_PATH=/tmp,LANG=en_us_8859_1,LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/tmp," + "PATH=$PATH:/tmp,NEW_PATH=$NEW_PATH:/tmp"); conf.set("path", System.getenv("PATH")); conf.set(mapTaskJavaOptsKey, mapTaskJavaOpts); From 83ab8f087b6a7554120ca5782aa28deae566d57c Mon Sep 17 00:00:00 2001 From: Mahadev Konar Date: Tue, 14 Feb 2012 22:49:00 +0000 Subject: [PATCH 2/6] MAPREDUCE-3858. Task attempt failure during commit results in task never completing. (Tom White via mahadev) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1244254 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../mapreduce/v2/app/job/impl/TaskImpl.java | 4 ++ .../v2/app/job/impl/TestTaskImpl.java | 56 +++++++++++++++++-- 3 files changed, 59 insertions(+), 4 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 74dacd22633..bb27716a751 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -815,6 +815,9 @@ Release 0.23.1 - 2012-02-08 MAPREDUCE-3802. Added test to validate that AM can crash multiple times and still can recover successfully after MAPREDUCE-3846. (vinodkv) + MAPREDUCE-3858. Task attempt failure during commit results in task never completing. + (Tom White via mahadev) + Release 0.23.0 - 2011-11-01 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java index e472e99cd21..0d7a2d8caee 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java @@ -832,6 +832,9 @@ public abstract class TaskImpl implements Task, EventHandler { public TaskState transition(TaskImpl task, TaskEvent event) { task.failedAttempts++; TaskTAttemptEvent castEvent = (TaskTAttemptEvent) event; + if (castEvent.getTaskAttemptID().equals(task.commitAttempt)) { + task.commitAttempt = null; + } TaskAttempt attempt = task.attempts.get(castEvent.getTaskAttemptID()); if (attempt.getAssignedContainerMgrAddress() != null) { //container was assigned @@ -877,6 +880,7 @@ public abstract class TaskImpl implements Task, EventHandler { protected void unSucceed(TaskImpl task) { ++task.numberUncompletedAttempts; + task.commitAttempt = null; task.successfulAttempt = null; } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java index dcc9b07cc38..0033528347e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java @@ -18,6 +18,8 @@ package org.apache.hadoop.mapreduce.v2.app.job.impl; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -261,6 +263,12 @@ public class TestTaskImpl { assertTaskRunningState(); } + private void commitTaskAttempt(TaskAttemptId attemptId) { + mockTask.handle(new TaskTAttemptEvent(attemptId, + TaskEventType.T_ATTEMPT_COMMIT_PENDING)); + assertTaskRunningState(); + } + private MockTaskAttemptImpl getLastAttempt() { return taskAttempts.get(taskAttempts.size()-1); } @@ -279,32 +287,45 @@ public class TestTaskImpl { assertTaskRunningState(); } + private void failRunningTaskAttempt(TaskAttemptId attemptId) { + mockTask.handle(new TaskTAttemptEvent(attemptId, + TaskEventType.T_ATTEMPT_FAILED)); + assertTaskRunningState(); + } + /** * {@link TaskState#NEW} */ private void assertTaskNewState() { - assertEquals(mockTask.getState(), TaskState.NEW); + assertEquals(TaskState.NEW, mockTask.getState()); } /** * {@link TaskState#SCHEDULED} */ private void assertTaskScheduledState() { - assertEquals(mockTask.getState(), TaskState.SCHEDULED); + assertEquals(TaskState.SCHEDULED, mockTask.getState()); } /** * {@link TaskState#RUNNING} */ private void assertTaskRunningState() { - assertEquals(mockTask.getState(), TaskState.RUNNING); + assertEquals(TaskState.RUNNING, mockTask.getState()); } /** * {@link TaskState#KILL_WAIT} */ private void assertTaskKillWaitState() { - assertEquals(mockTask.getState(), TaskState.KILL_WAIT); + assertEquals(TaskState.KILL_WAIT, mockTask.getState()); + } + + /** + * {@link TaskState#SUCCEEDED} + */ + private void assertTaskSucceededState() { + assertEquals(TaskState.SUCCEEDED, mockTask.getState()); } @Test @@ -409,5 +430,32 @@ public class TestTaskImpl { assert(mockTask.getProgress() == progress); } + + @Test + public void testFailureDuringTaskAttemptCommit() { + TaskId taskId = getNewTaskID(); + scheduleTaskAttempt(taskId); + launchTaskAttempt(getLastAttempt().getAttemptId()); + updateLastAttemptState(TaskAttemptState.COMMIT_PENDING); + commitTaskAttempt(getLastAttempt().getAttemptId()); + + // During the task attempt commit there is an exception which causes + // the attempt to fail + updateLastAttemptState(TaskAttemptState.FAILED); + failRunningTaskAttempt(getLastAttempt().getAttemptId()); + + assertEquals(2, taskAttempts.size()); + updateLastAttemptState(TaskAttemptState.SUCCEEDED); + commitTaskAttempt(getLastAttempt().getAttemptId()); + mockTask.handle(new TaskTAttemptEvent(getLastAttempt().getAttemptId(), + TaskEventType.T_ATTEMPT_SUCCEEDED)); + + assertFalse("First attempt should not commit", + mockTask.canCommit(taskAttempts.get(0).getAttemptId())); + assertTrue("Second attempt should commit", + mockTask.canCommit(getLastAttempt().getAttemptId())); + + assertTaskSucceededState(); + } } From 89bd2210a7535d72c36aec441e9b99392628386b Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Tue, 14 Feb 2012 23:09:05 +0000 Subject: [PATCH 3/6] MAPREDUCE-3736. Variable substitution depth too large for fs.default.name causes jobs to fail (ahmed via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1244264 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/src/main/resources/core-default.xml | 7 +++++++ hadoop-mapreduce-project/CHANGES.txt | 5 ++++- .../apache/hadoop/mapred/TestMRWithDistributedCache.java | 2 +- .../java/org/apache/hadoop/conf/TestNoDefaultsJobConf.java | 4 ++-- .../src/test/java/org/apache/hadoop/fs/JHLogAnalyzer.java | 2 +- .../src/test/java/org/apache/hadoop/io/FileBench.java | 2 +- .../apache/hadoop/mapred/TestCombineFileInputFormat.java | 2 +- .../hadoop/mapred/TestConcatenatedCompressedInput.java | 2 +- .../java/org/apache/hadoop/mapred/TestTextInputFormat.java | 2 +- .../org/apache/hadoop/mapreduce/TestMapCollection.java | 2 +- .../hadoop/mapreduce/lib/input/TestFileInputFormat.java | 4 ++-- .../mapreduce/lib/input/TestMRKeyValueTextInputFormat.java | 2 +- .../hadoop-yarn-common/src/main/resources/yarn-default.xml | 2 +- 13 files changed, 24 insertions(+), 14 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 9cf1eaf3116..9781e9aec08 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -216,6 +216,13 @@ determine the host, port, etc. for a filesystem. + + fs.default.name + file:/// + Deprecated. Use (fs.defaultFS) property + instead + + fs.trash.interval 0 diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index bb27716a751..22324a79f2c 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -106,7 +106,10 @@ Release 0.23.2 - UNRELEASED MAPREDUCE-3852. Test TestLinuxResourceCalculatorPlugin failing. (Thomas Graves via mahadev) - + + MAPREDUCE-3736. Variable substitution depth too large for fs.default.name + causes jobs to fail (ahmed via tucu). + Release 0.23.1 - 2012-02-08 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java index ed89bf9fd4f..6798831ba73 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java @@ -159,7 +159,7 @@ public class TestMRWithDistributedCache extends TestCase { public void testLocalJobRunner() throws Exception { Configuration c = new Configuration(); c.set(JTConfig.JT_IPC_ADDRESS, "local"); - c.set("fs.default.name", "file:///"); + c.set("fs.defaultFS", "file:///"); testWithConf(c); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestNoDefaultsJobConf.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestNoDefaultsJobConf.java index d91754d71d7..81c474e9cb6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestNoDefaultsJobConf.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/conf/TestNoDefaultsJobConf.java @@ -59,7 +59,7 @@ public class TestNoDefaultsJobConf extends HadoopTestCase { JobConf conf = new JobConf(false); - conf.set("fs.default.name", createJobConf().get("fs.default.name")); + conf.set("fs.defaultFS", createJobConf().get("fs.defaultFS")); conf.setJobName("mr"); @@ -100,4 +100,4 @@ public class TestNoDefaultsJobConf extends HadoopTestCase { } -} \ No newline at end of file +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/JHLogAnalyzer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/JHLogAnalyzer.java index 540f15bffa2..347dd066a98 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/JHLogAnalyzer.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/JHLogAnalyzer.java @@ -1024,7 +1024,7 @@ public class JHLogAnalyzer { if(testFile != null) { LOG.info("Start JHLA test ============ "); LocalFileSystem lfs = FileSystem.getLocal(conf); - conf.set("fs.default.name", "file:///"); + conf.set("fs.defaultFS", "file:///"); JHLAMapper map = new JHLAMapper(conf); map.parseLogFile(lfs, new Path(testFile), 0L, new LoggingCollector(), Reporter.NULL); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/io/FileBench.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/io/FileBench.java index fb2a4168959..f155daef141 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/io/FileBench.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/io/FileBench.java @@ -53,7 +53,7 @@ public class FileBench extends Configured implements Tool { "unless they are also explicitly included, as in \"-pln -zip\"\n" + "Note that CompressionType params only apply to SequenceFiles\n\n" + "Useful options to set:\n" + -"-D fs.default.name=\"file:///\" \\\n" + +"-D fs.defaultFS=\"file:///\" \\\n" + "-D fs.file.impl=org.apache.hadoop.fs.RawLocalFileSystem \\\n" + "-D filebench.file.bytes=$((10*1024*1024*1024)) \\\n" + "-D filebench.key.words=5 \\\n" + diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineFileInputFormat.java index e07577e1c41..218ef85a3ab 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineFileInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineFileInputFormat.java @@ -41,7 +41,7 @@ public class TestCombineFileInputFormat { private static FileSystem localFs = null; static { try { - defaultConf.set("fs.default.name", "file:///"); + defaultConf.set("fs.defaultFS", "file:///"); localFs = FileSystem.getLocal(defaultConf); } catch (IOException e) { throw new RuntimeException("init failure", e); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestConcatenatedCompressedInput.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestConcatenatedCompressedInput.java index 1192ee70ce5..df409c10be8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestConcatenatedCompressedInput.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestConcatenatedCompressedInput.java @@ -69,7 +69,7 @@ public class TestConcatenatedCompressedInput { static { try { - defaultConf.set("fs.default.name", "file:///"); + defaultConf.set("fs.defaultFS", "file:///"); localFs = FileSystem.getLocal(defaultConf); } catch (IOException e) { throw new RuntimeException("init failure", e); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextInputFormat.java index faadf795942..4066a8e55f8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextInputFormat.java @@ -55,7 +55,7 @@ public class TestTextInputFormat { private static FileSystem localFs = null; static { try { - defaultConf.set("fs.default.name", "file:///"); + defaultConf.set("fs.defaultFS", "file:///"); localFs = FileSystem.getLocal(defaultConf); } catch (IOException e) { throw new RuntimeException("init failure", e); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMapCollection.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMapCollection.java index 813bf9f1cc8..ecc01dbd02f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMapCollection.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMapCollection.java @@ -314,7 +314,7 @@ public class TestMapCollection { job.setNumReduceTasks(1); job.getConfiguration().set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME); job.getConfiguration().setInt(MRJobConfig.IO_SORT_FACTOR, 1000); - job.getConfiguration().set("fs.default.name", "file:///"); + job.getConfiguration().set("fs.defaultFS", "file:///"); job.getConfiguration().setInt("test.mapcollection.num.maps", 1); job.setInputFormatClass(FakeIF.class); job.setOutputFormatClass(NullOutputFormat.class); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFileInputFormat.java index 824e6842cff..28359585a2d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFileInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFileInputFormat.java @@ -45,9 +45,9 @@ public class TestFileInputFormat { @Test public void testAddInputPath() throws IOException { final Configuration conf = new Configuration(); - conf.set("fs.default.name", "s3://abc:xyz@hostname/"); + conf.set("fs.defaultFS", "s3://abc:xyz@hostname/"); final Job j = Job.getInstance(conf); - j.getConfiguration().set("fs.default.name", "s3://abc:xyz@hostname/"); + j.getConfiguration().set("fs.defaultFS", "s3://abc:xyz@hostname/"); //setup default fs final FileSystem defaultfs = FileSystem.get(conf); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMRKeyValueTextInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMRKeyValueTextInputFormat.java index ea3b83807ce..54ad498dd48 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMRKeyValueTextInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMRKeyValueTextInputFormat.java @@ -57,7 +57,7 @@ public class TestMRKeyValueTextInputFormat { private static FileSystem localFs = null; static { try { - defaultConf.set("fs.default.name", "file:///"); + defaultConf.set("fs.defaultFS", "file:///"); localFs = FileSystem.getLocal(defaultConf); } catch (IOException e) { throw new RuntimeException("init failure", e); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index cea45798a71..217d6e1c775 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -457,7 +457,7 @@ mapreduce.job.hdfs-servers - ${fs.default.name} + ${fs.defaultFS} From 063be5749df0e4ccb8f6fa4794b3d29a8f3a0afa Mon Sep 17 00:00:00 2001 From: Todd Lipcon Date: Wed, 15 Feb 2012 18:10:21 +0000 Subject: [PATCH 4/6] HADOOP-6502. Improve the performance of Configuration.getClassByName when the class is not found by caching negative results. Contributed by Sharad Agarwal and Todd Lipcon. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1244620 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 4 +++ .../org/apache/hadoop/conf/Configuration.java | 36 +++++++++++++++---- .../apache/hadoop/util/ReflectionUtils.java | 15 +++++--- 3 files changed, 44 insertions(+), 11 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 63f43189c8c..e6fe6e5c47e 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -169,6 +169,10 @@ Release 0.23.2 - UNRELEASED HADOOP-8071. Avoid an extra packet in client code when nagling is disabled. (todd) + HADOOP-6502. Improve the performance of Configuration.getClassByName when + the class is not found by caching negative results. + (sharad, todd via todd) + BUG FIXES HADOOP-8042 When copying a file out of HDFS, modifying it, and uploading diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index 84a31871195..d71aaf58106 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -1146,6 +1146,22 @@ public class Configuration implements Iterable>, * @throws ClassNotFoundException if the class is not found. */ public Class getClassByName(String name) throws ClassNotFoundException { + Class ret = getClassByNameOrNull(name); + if (ret == null) { + throw new ClassNotFoundException("Class " + name + " not found"); + } + return ret; + } + + /** + * Load a class by name, returning null rather than throwing an exception + * if it couldn't be loaded. This is to avoid the overhead of creating + * an exception. + * + * @param name the class name + * @return the class object, or null if it could not be found. + */ + public Class getClassByNameOrNull(String name) { Map> map; synchronized (CACHE_CLASSES) { @@ -1157,12 +1173,20 @@ public class Configuration implements Iterable>, } } - Class clazz = map.get(name); - if (clazz == null) { - clazz = Class.forName(name, true, classLoader); - if (clazz != null) { - // two putters can race here, but they'll put the same class - map.put(name, clazz); + Class clazz = null; + if (!map.containsKey(name)) { + try { + clazz = Class.forName(name, true, classLoader); + } catch (ClassNotFoundException e) { + map.put(name, null); //cache negative that class is not found + return null; + } + // two putters can race here, but they'll put the same class + map.put(name, clazz); + } else { // check already performed on this class name + clazz = map.get(name); + if (clazz == null) { // found the negative + return null; } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ReflectionUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ReflectionUtils.java index 0387c7e8b8e..fc5289779e6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ReflectionUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ReflectionUtils.java @@ -86,17 +86,22 @@ public class ReflectionUtils { //invoke configure on theObject try { Class jobConfClass = - conf.getClassByName("org.apache.hadoop.mapred.JobConf"); + conf.getClassByNameOrNull("org.apache.hadoop.mapred.JobConf"); + if (jobConfClass == null) { + return; + } + Class jobConfigurableClass = - conf.getClassByName("org.apache.hadoop.mapred.JobConfigurable"); - if (jobConfClass.isAssignableFrom(conf.getClass()) && + conf.getClassByNameOrNull("org.apache.hadoop.mapred.JobConfigurable"); + if (jobConfigurableClass == null) { + return; + } + if (jobConfClass.isAssignableFrom(conf.getClass()) && jobConfigurableClass.isAssignableFrom(theObject.getClass())) { Method configureMethod = jobConfigurableClass.getMethod("configure", jobConfClass); configureMethod.invoke(theObject, conf); } - } catch (ClassNotFoundException e) { - //JobConf/JobConfigurable not in classpath. no need to configure } catch (Exception e) { throw new RuntimeException("Error in configuring object", e); } From 3bd230af1199ad741518c2359618bb8fb93e6d2a Mon Sep 17 00:00:00 2001 From: Todd Lipcon Date: Wed, 15 Feb 2012 18:27:37 +0000 Subject: [PATCH 5/6] HDFS-2950. Secondary NN HTTPS address should be listed as a NAMESERVICE_SPECIFIC_KEY. Contributed by Todd Lipcon. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1244635 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 ++ .../main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java | 1 + .../java/org/apache/hadoop/hdfs/server/namenode/NameNode.java | 1 + .../apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java | 3 ++- 5 files changed, 9 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index f936c4612c6..11b67eb0d96 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -245,6 +245,9 @@ Release 0.23.2 - UNRELEASED HDFS-2815. Namenode sometimes oes not come out of safemode during NN crash + restart. (Uma Maheswara Rao via suresh) + HDFS-2950. Secondary NN HTTPS address should be listed as a + NAMESERVICE_SPECIFIC_KEY. (todd) + Release 0.23.1 - 2012-02-08 INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 98b57aa373c..fd1b33a26d2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -86,6 +86,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final int DFS_NAMENODE_SAFEMODE_MIN_DATANODES_DEFAULT = 0; public static final String DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY = "dfs.namenode.secondary.http-address"; public static final String DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_DEFAULT = "0.0.0.0:50090"; + public static final String DFS_NAMENODE_SECONDARY_HTTPS_PORT_KEY = "dfs.namenode.secondary.https-port"; + public static final int DFS_NAMENODE_SECONDARY_HTTPS_PORT_DEFAULT = 50490; public static final String DFS_NAMENODE_CHECKPOINT_CHECK_PERIOD_KEY = "dfs.namenode.checkpoint.check.period"; public static final long DFS_NAMENODE_CHECKPOINT_CHECK_PERIOD_DEFAULT = 60; public static final String DFS_NAMENODE_CHECKPOINT_PERIOD_KEY = "dfs.namenode.checkpoint.period"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java index 294e5852cbd..44533f13a12 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java @@ -81,6 +81,7 @@ public class HdfsConfiguration extends Configuration { deprecate("dfs.safemode.extension", DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY); deprecate("dfs.safemode.threshold.pct", DFSConfigKeys.DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_KEY); deprecate("dfs.secondary.http.address", DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY); + deprecate("dfs.secondary.https.port", DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTPS_PORT_KEY); deprecate("dfs.socket.timeout", DFSConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY); deprecate("fs.checkpoint.dir", DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_DIR_KEY); deprecate("fs.checkpoint.edits.dir", DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_EDITS_DIR_KEY); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index a5f11fab68b..7a51f2ed301 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -117,6 +117,7 @@ public class NameNode { DFS_NAMENODE_HTTPS_ADDRESS_KEY, DFS_NAMENODE_KEYTAB_FILE_KEY, DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY, + DFS_NAMENODE_SECONDARY_HTTPS_PORT_KEY, DFS_SECONDARY_NAMENODE_KEYTAB_FILE_KEY, DFS_NAMENODE_BACKUP_ADDRESS_KEY, DFS_NAMENODE_BACKUP_HTTP_ADDRESS_KEY, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java index f605f617527..50dca62e8fe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java @@ -266,7 +266,8 @@ public class SecondaryNameNode implements Runnable { Krb5AndCertsSslSocketConnector.KRB5_CIPHER_SUITES.get(0)); InetSocketAddress secInfoSocAddr = NetUtils.createSocketAddr(infoBindAddress + ":"+ conf.getInt( - "dfs.secondary.https.port", 443)); + DFS_NAMENODE_SECONDARY_HTTPS_PORT_KEY, + DFS_NAMENODE_SECONDARY_HTTPS_PORT_DEFAULT)); imagePort = secInfoSocAddr.getPort(); infoServer.addSslListener(secInfoSocAddr, conf, false, true); } From f82e862e1f1f2df2e43225280cb6b967eded097b Mon Sep 17 00:00:00 2001 From: Jitendra Nath Pandey Date: Wed, 15 Feb 2012 18:43:01 +0000 Subject: [PATCH 6/6] HDFS-2525. Race between BlockPoolSliceScanner and append. Contributed by Brandon Li. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1244640 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../datanode/BlockPoolSliceScanner.java | 20 ++++++++++++++----- .../hdfs/TestAppendDifferentChecksum.java | 6 ------ 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 11b67eb0d96..0ab243cc285 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -210,6 +210,9 @@ Trunk (unreleased changes) dfs.client.block.write.replace-datanode-on-failure.enable to be mistakenly disabled. (atm) + HDFS-2525. Race between BlockPoolSliceScanner and append. (Brandon Li + via jitendra) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java index e3709463b41..54c1b6f3952 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java @@ -51,11 +51,8 @@ import org.apache.hadoop.hdfs.util.DataTransferThrottler; import org.apache.hadoop.io.IOUtils; /** - * Performs two types of scanning: - *
  • Gets block files from the data directories and reconciles the - * difference between the blocks on the disk and in memory.
    • - *
  • Scans the data directories for block files under a block pool - * and verifies that the files are not corrupt
    • + * Scans the block files under a block pool and verifies that the + * files are not corrupt. * This keeps track of blocks and their last verification times. * Currently it does not modify the metadata for block. */ @@ -430,6 +427,19 @@ class BlockPoolSliceScanner { return; } + // If the block exists, the exception may due to a race with write: + // The BlockSender got an old block path in rbw. BlockReceiver removed + // the rbw block from rbw to finalized but BlockSender tried to open the + // file before BlockReceiver updated the VolumeMap. The state of the + // block can be changed again now, so ignore this error here. If there + // is a block really deleted by mistake, DirectoryScan should catch it. + if (e instanceof FileNotFoundException ) { + LOG.info("Verification failed for " + block + + ". It may be due to race with write."); + deleteBlock(block.getLocalBlock()); + return; + } + LOG.warn((second ? "Second " : "First ") + "Verification failed for " + block, e); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAppendDifferentChecksum.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAppendDifferentChecksum.java index f296419bde5..9fbb7605d44 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAppendDifferentChecksum.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAppendDifferentChecksum.java @@ -47,12 +47,6 @@ public class TestAppendDifferentChecksum { public static void setupCluster() throws IOException { Configuration conf = new HdfsConfiguration(); conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 4096); - - // disable block scanner, since otherwise this test can trigger - // HDFS-2525, which is a different bug than we're trying to unit test - // here! When HDFS-2525 is fixed, this can be removed. - conf.setInt(DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, -1); - conf.set("fs.hdfs.impl.disable.cache", "true"); cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(1)