MAPREDUCE-4899. Implemented a MR specific plugin for tracking finished applications that YARN's ResourceManager doesn't keep track off anymore. Contributed by Derek Dagit.

svn merge --ignore-ancestry -c 1425225 ../../trunk/


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1425226 13f79535-47bb-0310-9956-ffa450edef68

hadoop-mapreduce-project/CHANGES.txt

@@ -26,6 +26,10 @@ Release 2.0.3-alpha - Unreleased
     MAPREDUCE-4845. ClusterStatus.getMaxMemory() and getUsedMemory() exist in
     MR1 but not MR2. (Sandy Ryza via tomwhite)
 
+    MAPREDUCE-4899. Implemented a MR specific plugin for tracking finished
+    applications that YARN's ResourceManager doesn't keep track off anymore
+    (Derek Dagit via vinodkv)
+
   OPTIMIZATIONS
 
   BUG FIXES

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml

@@ -0,0 +1,46 @@
+<?xml version="1.0"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+                      http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <parent>
+    <artifactId>hadoop-mapreduce-client</artifactId>
+    <groupId>org.apache.hadoop</groupId>
+    <version>3.0.0-SNAPSHOT</version>
+  </parent>
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>org.apache.hadoop</groupId>
+  <artifactId>hadoop-mapreduce-client-hs-plugins</artifactId>
+  <version>3.0.0-SNAPSHOT</version>
+  <name>hadoop-mapreduce-client-hs-plugins</name>
+
+  <properties>
+    <!-- Needed for generating FindBugs warnings using parent pom -->
+    <mr.basedir>${project.parent.basedir}/../</mr.basedir>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-yarn-common</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-mapreduce-client-common</artifactId>
+    </dependency>
+  </dependencies>
+
+</project>

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/MapReduceTrackingUriPlugin.java

@@ -0,0 +1,61 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.hs.webapp;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.util.TrackingUriPlugin;
+
+
+public class MapReduceTrackingUriPlugin extends TrackingUriPlugin implements
+    Configurable {
+
+  @Override
+  public void setConf(Configuration conf) {
+    Configuration jobConf = null;
+    // Force loading of mapred configuration.
+    if (conf != null) {
+      jobConf = new JobConf(conf);
+    } else {
+      jobConf = new JobConf();
+    }
+    super.setConf(jobConf);
+  }
+
+  /**
+   * Gets the URI to access the given application on MapReduce history server
+   * @param id the ID for which a URI is returned
+   * @return the tracking URI
+   * @throws URISyntaxException
+   */
+  @Override
+  public URI getTrackingUri(ApplicationId id) throws URISyntaxException {
+    String jobSuffix = id.toString().replaceFirst("^application_", "job_");
+    String historyServerAddress =
+        this.getConf().get(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS);
+    return new URI("http://" + historyServerAddress + "/jobhistory/job/"
+        + jobSuffix);
+  }
+}

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestMapReduceTrackingUriPlugin.java

@@ -0,0 +1,47 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.hs.webapp;
+
+import static org.junit.Assert.*;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.util.BuilderUtils;
+import org.junit.Test;
+
+public class TestMapReduceTrackingUriPlugin {
+  @Test
+  public void testProducesHistoryServerUriForAppId() throws URISyntaxException {
+    final String historyAddress = "example.net:424242";
+    YarnConfiguration conf = new YarnConfiguration();
+    conf.set(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS, historyAddress);
+    MapReduceTrackingUriPlugin plugin = new MapReduceTrackingUriPlugin();
+    plugin.setConf(conf);
+    ApplicationId id = BuilderUtils.newApplicationId(6384623l, 5);
+    String jobSuffix = id.toString().replaceFirst("^application_", "job_");
+    URI expected =
+        new URI("http://" + historyAddress + "/jobhistory/job/" + jobSuffix);
+    URI actual = plugin.getTrackingUri(id);
+    assertEquals(expected, actual);
+  }
+}

hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml

@@ -194,5 +194,6 @@
     <module>hadoop-mapreduce-client-app</module>
     <module>hadoop-mapreduce-client-jobclient</module>
     <module>hadoop-mapreduce-client-hs</module>
+    <module>hadoop-mapreduce-client-hs-plugins</module>
   </modules>
 </project>