From abfb2957d19c43d2261a94dcff109defc269f470 Mon Sep 17 00:00:00 2001 From: Wangda Tan Date: Tue, 15 Mar 2016 17:28:40 -0700 Subject: [PATCH] MAPREDUCE-6579. JobStatus#getFailureInfo should not output diagnostic information when the job is running. (Akira AJISAKA via wangda) (cherry picked from commit 6529c87551e07e9da1e1b273eff4c6bad6b3e838) --- .../apache/hadoop/mapred/ClusterStatus.java | 35 +++++++- .../apache/hadoop/mapred/NotRunningJob.java | 7 +- .../hadoop/mapred/TestNetworkedJob.java | 89 +++++++++---------- 3 files changed, 84 insertions(+), 47 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ClusterStatus.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ClusterStatus.java index 904897b1326..c87b583c4ff 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ClusterStatus.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ClusterStatus.java @@ -171,7 +171,40 @@ public class ClusterStatus implements Writable { sb.append(blackListReport.replace("\n", ":")); return sb.toString(); } - + + @Override + public int hashCode() { + int result = trackerName != null ? trackerName.hashCode() : 0; + result = 31 * result + (reasonForBlackListing != null ? + reasonForBlackListing.hashCode() : 0); + result = 31 * result + (blackListReport != null ? + blackListReport.hashCode() : 0); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + final BlackListInfo that = (BlackListInfo) obj; + if (trackerName == null ? that.trackerName != null : + !trackerName.equals(that.trackerName)) { + return false; + } + if (reasonForBlackListing == null ? that.reasonForBlackListing != null : + !reasonForBlackListing.equals(that.reasonForBlackListing)) { + return false; + } + if (blackListReport == null ? that.blackListReport != null : + !blackListReport.equals(that.blackListReport)) { + return false; + } + return true; + } } public static final long UNINITIALIZED_MEMORY_VALUE = -1; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java index 03552e4f2bc..90635a68854 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java @@ -137,7 +137,12 @@ public class NotRunningJob implements MRClientProtocol { jobReport.setJobState(jobState); jobReport.setUser(applicationReport.getUser()); jobReport.setStartTime(applicationReport.getStartTime()); - jobReport.setDiagnostics(applicationReport.getDiagnostics()); + YarnApplicationState state = applicationReport.getYarnApplicationState(); + if (state == YarnApplicationState.KILLED + || state == YarnApplicationState.FAILED + || state == YarnApplicationState.FINISHED) { + jobReport.setDiagnostics(applicationReport.getDiagnostics()); + } jobReport.setJobName(applicationReport.getName()); jobReport.setTrackingUrl(applicationReport.getTrackingUrl()); jobReport.setFinishTime(applicationReport.getFinishTime()); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestNetworkedJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestNetworkedJob.java index 2e0887e50db..45c7d1f79df 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestNetworkedJob.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestNetworkedJob.java @@ -155,9 +155,9 @@ public class TestNetworkedJob { // test getters assertTrue(runningJob.getConfiguration().toString() .endsWith("0001/job.xml")); - assertEquals(runningJob.getID(), jobId); - assertEquals(runningJob.getJobID(), jobId.toString()); - assertEquals(runningJob.getJobName(), "N/A"); + assertEquals(jobId, runningJob.getID()); + assertEquals(jobId.toString(), runningJob.getJobID()); + assertEquals("N/A", runningJob.getJobName()); assertTrue(runningJob.getJobFile().endsWith( ".staging/" + runningJob.getJobID() + "/job.xml")); assertTrue(runningJob.getTrackingURL().length() > 0); @@ -169,40 +169,40 @@ public class TestNetworkedJob { TaskCompletionEvent[] tce = runningJob.getTaskCompletionEvents(0); assertEquals(tce.length, 0); - assertEquals(runningJob.getHistoryUrl(),""); + assertEquals("", runningJob.getHistoryUrl()); assertFalse(runningJob.isRetired()); - assertEquals( runningJob.getFailureInfo(),""); - assertEquals(runningJob.getJobStatus().getJobName(), "N/A"); - assertEquals(client.getMapTaskReports(jobId).length, 0); + assertEquals("", runningJob.getFailureInfo()); + assertEquals("N/A", runningJob.getJobStatus().getJobName()); + assertEquals(0, client.getMapTaskReports(jobId).length); try { client.getSetupTaskReports(jobId); } catch (YarnRuntimeException e) { - assertEquals(e.getMessage(), "Unrecognized task type: JOB_SETUP"); + assertEquals("Unrecognized task type: JOB_SETUP", e.getMessage()); } try { client.getCleanupTaskReports(jobId); } catch (YarnRuntimeException e) { - assertEquals(e.getMessage(), "Unrecognized task type: JOB_CLEANUP"); + assertEquals("Unrecognized task type: JOB_CLEANUP", e.getMessage()); } - assertEquals(client.getReduceTaskReports(jobId).length, 0); + assertEquals(0, client.getReduceTaskReports(jobId).length); // test ClusterStatus ClusterStatus status = client.getClusterStatus(true); - assertEquals(status.getActiveTrackerNames().size(), 2); + assertEquals(2, status.getActiveTrackerNames().size()); // it method does not implemented and always return empty array or null; - assertEquals(status.getBlacklistedTrackers(), 0); - assertEquals(status.getBlacklistedTrackerNames().size(), 0); - assertEquals(status.getBlackListedTrackersInfo().size(), 0); - assertEquals(status.getJobTrackerStatus(), JobTrackerStatus.RUNNING); - assertEquals(status.getMapTasks(), 1); - assertEquals(status.getMaxMapTasks(), 20); - assertEquals(status.getMaxReduceTasks(), 4); - assertEquals(status.getNumExcludedNodes(), 0); - assertEquals(status.getReduceTasks(), 1); - assertEquals(status.getTaskTrackers(), 2); - assertEquals(status.getTTExpiryInterval(), 0); - assertEquals(status.getJobTrackerStatus(), JobTrackerStatus.RUNNING); - assertEquals(status.getGraylistedTrackers(), 0); + assertEquals(0, status.getBlacklistedTrackers()); + assertEquals(0, status.getBlacklistedTrackerNames().size()); + assertEquals(0, status.getBlackListedTrackersInfo().size()); + assertEquals(JobTrackerStatus.RUNNING, status.getJobTrackerStatus()); + assertEquals(1, status.getMapTasks()); + assertEquals(20, status.getMaxMapTasks()); + assertEquals(4, status.getMaxReduceTasks()); + assertEquals(0, status.getNumExcludedNodes()); + assertEquals(1, status.getReduceTasks()); + assertEquals(2, status.getTaskTrackers()); + assertEquals(0, status.getTTExpiryInterval()); + assertEquals(JobTrackerStatus.RUNNING, status.getJobTrackerStatus()); + assertEquals(0, status.getGraylistedTrackers()); // test read and write ByteArrayOutputStream dataOut = new ByteArrayOutputStream(); @@ -219,34 +219,34 @@ public class TestNetworkedJob { // test taskStatusfilter JobClient.setTaskOutputFilter(job, TaskStatusFilter.ALL); - assertEquals(JobClient.getTaskOutputFilter(job), TaskStatusFilter.ALL); + assertEquals(TaskStatusFilter.ALL, JobClient.getTaskOutputFilter(job)); // runningJob.setJobPriority(JobPriority.HIGH.name()); // test default map - assertEquals(client.getDefaultMaps(), 20); - assertEquals(client.getDefaultReduces(), 4); - assertEquals(client.getSystemDir().getName(), "jobSubmitDir"); + assertEquals(20, client.getDefaultMaps()); + assertEquals(4, client.getDefaultReduces()); + assertEquals("jobSubmitDir", client.getSystemDir().getName()); // test queue information JobQueueInfo[] rootQueueInfo = client.getRootQueues(); - assertEquals(rootQueueInfo.length, 1); - assertEquals(rootQueueInfo[0].getQueueName(), "default"); + assertEquals(1, rootQueueInfo.length); + assertEquals("default", rootQueueInfo[0].getQueueName()); JobQueueInfo[] qinfo = client.getQueues(); - assertEquals(qinfo.length, 1); - assertEquals(qinfo[0].getQueueName(), "default"); - assertEquals(client.getChildQueues("default").length, 0); - assertEquals(client.getJobsFromQueue("default").length, 1); + assertEquals(1, qinfo.length); + assertEquals("default", qinfo[0].getQueueName()); + assertEquals(0, client.getChildQueues("default").length); + assertEquals(1, client.getJobsFromQueue("default").length); assertTrue(client.getJobsFromQueue("default")[0].getJobFile().endsWith( "/job.xml")); JobQueueInfo qi = client.getQueueInfo("default"); - assertEquals(qi.getQueueName(), "default"); - assertEquals(qi.getQueueState(), "running"); + assertEquals("default", qi.getQueueName()); + assertEquals("running", qi.getQueueState()); QueueAclsInfo[] aai = client.getQueueAclsForCurrentUser(); - assertEquals(aai.length, 2); - assertEquals(aai[0].getQueueName(), "root"); - assertEquals(aai[1].getQueueName(), "default"); + assertEquals(2, aai.length); + assertEquals("root", aai[0].getQueueName()); + assertEquals("default", aai[1].getQueueName()); // test JobClient // The following asserts read JobStatus twice and ensure the returned @@ -282,12 +282,11 @@ public class TestNetworkedJob { BlackListInfo info2 = new BlackListInfo(); info2.readFields(new DataInputStream(new ByteArrayInputStream(byteOut .toByteArray()))); - assertEquals(info, info); - assertEquals(info.toString(), info.toString()); - assertEquals(info.getTrackerName(), "trackerName"); - assertEquals(info.getReasonForBlackListing(), "reasonForBlackListing"); - assertEquals(info.getBlackListReport(), "blackListInfo"); - + assertEquals(info, info2); + assertEquals(info.toString(), info2.toString()); + assertEquals("trackerName", info2.getTrackerName()); + assertEquals("reasonForBlackListing", info2.getReasonForBlackListing()); + assertEquals("blackListInfo", info2.getBlackListReport()); } /** * test run from command line JobQueueClient