MAPREDUCE-4899. Implemented a MR specific plugin for tracking finished applications that YARN's ResourceManager doesn't keep track off anymore. Contributed by Derek Dagit.

svn merge --ignore-ancestry -c 1425225 ../../trunk/


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1425226 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Vinod Kumar Vavilapalli 2012-12-22 01:49:02 +00:00
parent 5b6115e3bb
commit 121a79c7a1
5 changed files with 159 additions and 0 deletions

View File

@ -26,6 +26,10 @@ Release 2.0.3-alpha - Unreleased
MAPREDUCE-4845. ClusterStatus.getMaxMemory() and getUsedMemory() exist in
MR1 but not MR2. (Sandy Ryza via tomwhite)
MAPREDUCE-4899. Implemented a MR specific plugin for tracking finished
applications that YARN's ResourceManager doesn't keep track off anymore
(Derek Dagit via vinodkv)
OPTIMIZATIONS
BUG FIXES

View File

@ -0,0 +1,46 @@
<?xml version="1.0"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
<version>3.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-hs-plugins</artifactId>
<version>3.0.0-SNAPSHOT</version>
<name>hadoop-mapreduce-client-hs-plugins</name>
<properties>
<!-- Needed for generating FindBugs warnings using parent pom -->
<mr.basedir>${project.parent.basedir}/../</mr.basedir>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-common</artifactId>
</dependency>
</dependencies>
</project>

View File

@ -0,0 +1,61 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.mapreduce.v2.hs.webapp;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.util.TrackingUriPlugin;
public class MapReduceTrackingUriPlugin extends TrackingUriPlugin implements
Configurable {
@Override
public void setConf(Configuration conf) {
Configuration jobConf = null;
// Force loading of mapred configuration.
if (conf != null) {
jobConf = new JobConf(conf);
} else {
jobConf = new JobConf();
}
super.setConf(jobConf);
}
/**
* Gets the URI to access the given application on MapReduce history server
* @param id the ID for which a URI is returned
* @return the tracking URI
* @throws URISyntaxException
*/
@Override
public URI getTrackingUri(ApplicationId id) throws URISyntaxException {
String jobSuffix = id.toString().replaceFirst("^application_", "job_");
String historyServerAddress =
this.getConf().get(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS);
return new URI("http://" + historyServerAddress + "/jobhistory/job/"
+ jobSuffix);
}
}

View File

@ -0,0 +1,47 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.mapreduce.v2.hs.webapp;
import static org.junit.Assert.*;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.junit.Test;
public class TestMapReduceTrackingUriPlugin {
@Test
public void testProducesHistoryServerUriForAppId() throws URISyntaxException {
final String historyAddress = "example.net:424242";
YarnConfiguration conf = new YarnConfiguration();
conf.set(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS, historyAddress);
MapReduceTrackingUriPlugin plugin = new MapReduceTrackingUriPlugin();
plugin.setConf(conf);
ApplicationId id = BuilderUtils.newApplicationId(6384623l, 5);
String jobSuffix = id.toString().replaceFirst("^application_", "job_");
URI expected =
new URI("http://" + historyAddress + "/jobhistory/job/" + jobSuffix);
URI actual = plugin.getTrackingUri(id);
assertEquals(expected, actual);
}
}

View File

@ -194,5 +194,6 @@
<module>hadoop-mapreduce-client-app</module>
<module>hadoop-mapreduce-client-jobclient</module>
<module>hadoop-mapreduce-client-hs</module>
<module>hadoop-mapreduce-client-hs-plugins</module>
</modules>
</project>