From 50993e9d48dfa9b374a8ad456dae4b08e0e759c9 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Wed, 31 Oct 2012 19:13:03 +0000 Subject: [PATCH 1/2] MAPREDUCE-4724. job history web ui applications page should be sorted to display last app first (tgraves via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1404305 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsView.java | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 8289abfdb7b..b2f9a793636 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -620,6 +620,9 @@ Release 0.23.5 - UNRELEASED MAPREDUCE-4748. Invalid event: T_ATTEMPT_SUCCEEDED at SUCCEEDED. (jlowe) + MAPREDUCE-4724. job history web ui applications page should be sorted to + display last app first (tgraves via bobby) + Release 0.23.4 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsView.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsView.java index 630c541fc97..23e5d9120f6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsView.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsView.java @@ -85,7 +85,7 @@ protected Class content() { private String jobsTableInit() { return tableInit(). // Sort by id upon page load - append(", aaSorting: [[2, 'asc']]"). + append(", aaSorting: [[2, 'desc']]"). append(", aoColumnDefs:["). // Maps Total append("{'sType':'numeric', 'bSearchable': false, 'aTargets': [ 7 ] }"). From a124297cf016439ee426d3142627606875b9667a Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Thu, 1 Nov 2012 00:32:37 +0000 Subject: [PATCH 2/2] YARN-189. Fixed a deadlock between RM's ApplicationMasterService and the dispatcher. Contributed by Thomas Graves. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1404431 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../ApplicationMasterService.java | 29 ++++++++++++------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 9dbb6d2b5fe..86b5336d503 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -196,6 +196,9 @@ Release 0.23.5 - UNRELEASED YARN-166. capacity scheduler doesn't allow capacity < 1.0 (tgraves via bobby) + YARN-189. Fixed a deadlock between RM's ApplicationMasterService and the + dispatcher. (Thomas Graves via vinodkv) + Release 0.23.4 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java index 849111e54b5..96ee551e205 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java @@ -265,10 +265,10 @@ public AllocateResponse allocate(AllocateRequest request) // Oh damn! Sending reboot isn't enough. RM state is corrupted. TODO: allocateResponse.setAMResponse(reboot); return allocateResponse; - } - + } + // Allow only one thread in AM to do heartbeat at a time. - synchronized (lastResponse) { // BUG TODO: Locking order is screwed. + synchronized (lastResponse) { // Send the status update to the appAttempt. this.rmContext.getDispatcher().getEventHandler().handle( @@ -282,7 +282,8 @@ public AllocateResponse allocate(AllocateRequest request) Allocation allocation = this.rScheduler.allocate(appAttemptId, ask, release); - RMApp app = this.rmContext.getRMApps().get(appAttemptId.getApplicationId()); + RMApp app = this.rmContext.getRMApps().get( + appAttemptId.getApplicationId()); RMAppAttempt appAttempt = app.getRMAppAttempt(appAttemptId); AMResponse response = recordFactory.newRecordInstance(AMResponse.class); @@ -316,7 +317,18 @@ public AllocateResponse allocate(AllocateRequest request) .pullJustFinishedContainers()); response.setResponseId(lastResponse.getResponseId() + 1); response.setAvailableResources(allocation.getResourceLimit()); - responseMap.put(appAttemptId, response); + + AMResponse oldResponse = responseMap.put(appAttemptId, response); + if (oldResponse == null) { + // appAttempt got unregistered, remove it back out + responseMap.remove(appAttemptId); + String message = "App Attempt removed from the cache during allocate" + + appAttemptId; + LOG.error(message); + allocateResponse.setAMResponse(reboot); + return allocateResponse; + } + allocateResponse.setAMResponse(response); allocateResponse.setNumClusterNodes(this.rScheduler.getNumClusterNodes()); return allocateResponse; @@ -331,12 +343,7 @@ public void registerAppAttempt(ApplicationAttemptId attemptId) { } public void unregisterAttempt(ApplicationAttemptId attemptId) { - AMResponse lastResponse = responseMap.get(attemptId); - if (lastResponse != null) { - synchronized (lastResponse) { - responseMap.remove(attemptId); - } - } + responseMap.remove(attemptId); } public void refreshServiceAcls(Configuration configuration,