From 121a79c7a1807378d3049bb35da059e4926c56d1 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Sat, 22 Dec 2012 01:49:02 +0000 Subject: [PATCH] MAPREDUCE-4899. Implemented a MR specific plugin for tracking finished applications that YARN's ResourceManager doesn't keep track off anymore. Contributed by Derek Dagit. svn merge --ignore-ancestry -c 1425225 ../../trunk/ git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1425226 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 4 ++ .../pom.xml | 46 ++++++++++++++ .../hs/webapp/MapReduceTrackingUriPlugin.java | 61 +++++++++++++++++++ .../TestMapReduceTrackingUriPlugin.java | 47 ++++++++++++++ .../hadoop-mapreduce-client/pom.xml | 1 + 5 files changed, 159 insertions(+) create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/MapReduceTrackingUriPlugin.java create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestMapReduceTrackingUriPlugin.java diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 96b176614df..09d14995859 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -26,6 +26,10 @@ Release 2.0.3-alpha - Unreleased MAPREDUCE-4845. ClusterStatus.getMaxMemory() and getUsedMemory() exist in MR1 but not MR2. (Sandy Ryza via tomwhite) + MAPREDUCE-4899. Implemented a MR specific plugin for tracking finished + applications that YARN's ResourceManager doesn't keep track off anymore + (Derek Dagit via vinodkv) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml new file mode 100644 index 00000000000..e1648eb7497 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml @@ -0,0 +1,46 @@ + + + + + hadoop-mapreduce-client + org.apache.hadoop + 3.0.0-SNAPSHOT + + 4.0.0 + org.apache.hadoop + hadoop-mapreduce-client-hs-plugins + 3.0.0-SNAPSHOT + hadoop-mapreduce-client-hs-plugins + + + + ${project.parent.basedir}/../ + + + + + org.apache.hadoop + hadoop-yarn-common + + + org.apache.hadoop + hadoop-mapreduce-client-common + + + + diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/MapReduceTrackingUriPlugin.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/MapReduceTrackingUriPlugin.java new file mode 100644 index 00000000000..6d148a7825c --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/MapReduceTrackingUriPlugin.java @@ -0,0 +1,61 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.mapreduce.v2.hs.webapp; + +import java.net.URI; +import java.net.URISyntaxException; + +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.util.TrackingUriPlugin; + + +public class MapReduceTrackingUriPlugin extends TrackingUriPlugin implements + Configurable { + + @Override + public void setConf(Configuration conf) { + Configuration jobConf = null; + // Force loading of mapred configuration. + if (conf != null) { + jobConf = new JobConf(conf); + } else { + jobConf = new JobConf(); + } + super.setConf(jobConf); + } + + /** + * Gets the URI to access the given application on MapReduce history server + * @param id the ID for which a URI is returned + * @return the tracking URI + * @throws URISyntaxException + */ + @Override + public URI getTrackingUri(ApplicationId id) throws URISyntaxException { + String jobSuffix = id.toString().replaceFirst("^application_", "job_"); + String historyServerAddress = + this.getConf().get(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS); + return new URI("http://" + historyServerAddress + "/jobhistory/job/" + + jobSuffix); + } +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestMapReduceTrackingUriPlugin.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestMapReduceTrackingUriPlugin.java new file mode 100644 index 00000000000..1b5cea9ff2f --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestMapReduceTrackingUriPlugin.java @@ -0,0 +1,47 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.mapreduce.v2.hs.webapp; + +import static org.junit.Assert.*; + +import java.net.URI; +import java.net.URISyntaxException; + +import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.util.BuilderUtils; +import org.junit.Test; + +public class TestMapReduceTrackingUriPlugin { + @Test + public void testProducesHistoryServerUriForAppId() throws URISyntaxException { + final String historyAddress = "example.net:424242"; + YarnConfiguration conf = new YarnConfiguration(); + conf.set(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS, historyAddress); + MapReduceTrackingUriPlugin plugin = new MapReduceTrackingUriPlugin(); + plugin.setConf(conf); + ApplicationId id = BuilderUtils.newApplicationId(6384623l, 5); + String jobSuffix = id.toString().replaceFirst("^application_", "job_"); + URI expected = + new URI("http://" + historyAddress + "/jobhistory/job/" + jobSuffix); + URI actual = plugin.getTrackingUri(id); + assertEquals(expected, actual); + } +} \ No newline at end of file diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml index b0bd4c0b76c..9d0ce0e6400 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml @@ -194,5 +194,6 @@ hadoop-mapreduce-client-app hadoop-mapreduce-client-jobclient hadoop-mapreduce-client-hs + hadoop-mapreduce-client-hs-plugins