Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

MAPREDUCE-3649. Job End notification gives an error on calling back. (Ravi Prakash via mahadev) 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1232126 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Mahadev Konar 2012-01-16 19:35:06 +00:00
parent a9002bfea1
commit a24339e50d
2 changed files with 21 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -493,6 +493,9 @@ Release 0.23.1 - Unreleased
MAPREDUCE-3404. Corrected MR AM to honor speculative configuration and enable MAPREDUCE-3404. Corrected MR AM to honor speculative configuration and enable
speculating either maps or reduces. (Eric Payne via vinodkv) speculating either maps or reduces. (Eric Payne via vinodkv)
MAPREDUCE-3649. Job End notification gives an error on calling back.
(Ravi Prakash via mahadev)
Release 0.23.0 - 2011-11-01 Release 0.23.0 - 2011-11-01
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

32
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java
View File

@ -19,12 +19,11 @@
package org.apache.hadoop.mapreduce.v2.app; package org.apache.hadoop.mapreduce.v2.app;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.net.HttpURLConnection;
import java.net.InetSocketAddress; import java.net.InetSocketAddress;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.net.Proxy; import java.net.Proxy;
import java.net.URL;
import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
@ -40,7 +39,8 @@ import org.mortbay.log.Log;
* User can specify number of retry attempts and a time interval at which to * User can specify number of retry attempts and a time interval at which to
* attempt retries</li><li> * attempt retries</li><li>
* Cluster administrators can set final parameters to set maximum number of * Cluster administrators can set final parameters to set maximum number of
* tries (0 would disable job end notification) and max time interval</li><li> * tries (0 would disable job end notification) and max time interval and a
* proxy if needed</li><li>
* The URL may contain sentinels which will be replaced by jobId and jobStatus * The URL may contain sentinels which will be replaced by jobId and jobStatus
* (eg. SUCCEEDED/KILLED/FAILED) </li> </ul> * (eg. SUCCEEDED/KILLED/FAILED) </li> </ul>
* </p> * </p>
@ -59,8 +59,8 @@ public class JobEndNotifier implements Configurable {
/** /**
* Parse the URL that needs to be notified of the end of the job, along * Parse the URL that needs to be notified of the end of the job, along
* with the number of retries in case of failure and the amount of time to * with the number of retries in case of failure, the amount of time to
* wait between retries * wait between retries and proxy settings
* @param conf the configuration * @param conf the configuration
*/ */
public void setConf(Configuration conf) { public void setConf(Configuration conf) {
@ -119,15 +119,19 @@ public class JobEndNotifier implements Configurable {
boolean success = false; boolean success = false;
try { try {
Log.info("Job end notification trying " + urlToNotify); Log.info("Job end notification trying " + urlToNotify);
URLConnection conn = urlToNotify.openConnection(proxyToUse); HttpURLConnection conn = (HttpURLConnection) urlToNotify.openConnection();
conn.setConnectTimeout(5*1000); conn.setConnectTimeout(5*1000);
conn.setReadTimeout(5*1000); conn.setReadTimeout(5*1000);
conn.setAllowUserInteraction(false); conn.setAllowUserInteraction(false);
InputStream is = conn.getInputStream(); if(conn.getResponseCode() != HttpURLConnection.HTTP_OK) {
conn.getContent(); Log.warn("Job end notification to " + urlToNotify +" failed with code: "
is.close(); + conn.getResponseCode() + " and message \"" + conn.getResponseMessage()
success = true; +"\"");
Log.info("Job end notification to " + urlToNotify + " succeeded"); }
else {
success = true;
Log.info("Job end notification to " + urlToNotify + " succeeded");
}
} catch(IOException ioe) { } catch(IOException ioe) {
Log.warn("Job end notification to " + urlToNotify + " failed", ioe); Log.warn("Job end notification to " + urlToNotify + " failed", ioe);
} }
@ -135,8 +139,8 @@ public class JobEndNotifier implements Configurable {
} }
/** /**
* Notify a server of the completion of a submitted job. The server must have * Notify a server of the completion of a submitted job. The user must have
* configured MRConfig.JOB_END_NOTIFICATION_URLS * configured MRJobConfig.MR_JOB_END_NOTIFICATION_URL
* @param jobReport JobReport used to read JobId and JobStatus * @param jobReport JobReport used to read JobId and JobStatus
* @throws InterruptedException * @throws InterruptedException
*/ */