From fc62f00ffc1ba7ef884ed5d3182da43a389f21fa Mon Sep 17 00:00:00 2001 From: Arun Murthy Date: Wed, 28 Dec 2011 18:25:13 +0000 Subject: [PATCH] Merge -c 1225259 from trunk to branch-0.23 to fix MAPREDUCE-3604. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1225261 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 4 +- .../apache/hadoop/streaming/StreamUtil.java | 17 ++++--- .../hadoop/streaming/TestMRFramework.java | 48 +++++++++++++++++++ 3 files changed, 62 insertions(+), 7 deletions(-) create mode 100644 hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMRFramework.java diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index dd07d459ccc..40f169d948a 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -315,7 +315,9 @@ Release 0.23.1 - Unreleased MAPREDUCE-3586. Modified CompositeService to avoid duplicate stop operations thereby solving race conditions in MR AM shutdown. (vinodkv) ->>>>>>> .merge-right.r1221950 + MAPREDUCE-3604. Fixed streaming to use new mapreduce.framework.name to + check for local mode. (acmurthy) + Release 0.23.0 - 2011-11-01 INCOMPATIBLE CHANGES diff --git a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamUtil.java b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamUtil.java index 281e1df0a6a..7e8d0a2ac44 100644 --- a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamUtil.java +++ b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamUtil.java @@ -18,16 +18,19 @@ package org.apache.hadoop.streaming; -import java.io.*; -import java.net.*; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.net.InetAddress; +import java.net.URL; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; +import org.apache.hadoop.mapreduce.MRConfig; /** * Utilities used in streaming @@ -189,6 +192,8 @@ public class StreamUtil { } public static boolean isLocalJobTracker(JobConf job) { - return job.get(JTConfig.JT_IPC_ADDRESS, "local").equals("local"); + String framework = + job.get(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME); + return framework.equals(MRConfig.LOCAL_FRAMEWORK_NAME); } } diff --git a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMRFramework.java b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMRFramework.java new file mode 100644 index 00000000000..17dc7ad633d --- /dev/null +++ b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMRFramework.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.streaming; + +import static org.junit.Assert.*; + +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapreduce.MRConfig; +import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; +import org.junit.Test; + +public class TestMRFramework { + + @Test + public void testFramework() { + JobConf jobConf = new JobConf(); + jobConf.set(JTConfig.JT_IPC_ADDRESS, MRConfig.LOCAL_FRAMEWORK_NAME); + jobConf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME); + assertFalse("Expected 'isLocal' to be false", + StreamUtil.isLocalJobTracker(jobConf)); + + jobConf.set(JTConfig.JT_IPC_ADDRESS, MRConfig.LOCAL_FRAMEWORK_NAME); + jobConf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME); + assertFalse("Expected 'isLocal' to be false", + StreamUtil.isLocalJobTracker(jobConf)); + + jobConf.set(JTConfig.JT_IPC_ADDRESS, "jthost:9090"); + jobConf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME); + assertTrue("Expected 'isLocal' to be true", + StreamUtil.isLocalJobTracker(jobConf)); + } + +}