diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java index c039514a33f..9e9ec3cd2c9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java @@ -25,7 +25,7 @@ import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.util.Records; -import java.util.List; +import java.util.Map; import java.util.Set; /** @@ -451,10 +451,10 @@ public abstract void setLogAggregationStatus( @Public @Unstable - public abstract List getApplicationTimeouts(); + public abstract Map getApplicationTimeouts(); @Private @Unstable public abstract void setApplicationTimeouts( - List timeouts); + Map timeouts); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index af7d47a6d5d..db72f7442e7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -214,7 +214,12 @@ message ApplicationReportProto { optional PriorityProto priority = 23; optional string appNodeLabelExpression = 24; optional string amNodeLabelExpression = 25; - repeated ApplicationTimeoutProto application_timeouts = 26; + repeated AppTimeoutsMapProto appTimeouts = 26; +} + +message AppTimeoutsMapProto { + optional ApplicationTimeoutTypeProto application_timeout_type = 1; + optional ApplicationTimeoutProto application_timeout = 2; } message ApplicationTimeoutProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index 20a65bfba4c..efe5921ce3d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -714,7 +714,8 @@ private int printApplicationReport(String applicationId) appReportStr.println(appReport.getAppNodeLabelExpression()); appReportStr.print("\tAM container Node Label Expression : "); appReportStr.println(appReport.getAmNodeLabelExpression()); - for (ApplicationTimeout timeout : appReport.getApplicationTimeouts()) { + for (ApplicationTimeout timeout : appReport.getApplicationTimeouts() + .values()) { appReportStr.print("\tTimeoutType : " + timeout.getTimeoutType()); appReportStr.print("\tExpiryTime : " + timeout.getExpiryTime()); appReportStr.println( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index d08db570fd2..15ec6b50107 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -130,8 +130,8 @@ public void testGetApplicationReport() throws Exception { newApplicationReport.setPriority(Priority.newInstance(0)); ApplicationTimeout timeout = ApplicationTimeout .newInstance(ApplicationTimeoutType.LIFETIME, "UNLIMITED", -1); - newApplicationReport - .setApplicationTimeouts(Collections.singletonList(timeout)); + newApplicationReport.setApplicationTimeouts( + Collections.singletonMap(timeout.getTimeoutType(), timeout)); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport); @@ -2104,7 +2104,8 @@ public void testUpdateApplicationTimeout() throws Exception { "N/A", 0.53789f, "YARN", null); ApplicationTimeout timeout = ApplicationTimeout .newInstance(ApplicationTimeoutType.LIFETIME, "N/A", -1); - appReport.setApplicationTimeouts(Collections.singletonList(timeout)); + appReport.setApplicationTimeouts( + Collections.singletonMap(timeout.getTimeoutType(), timeout)); when(client.getApplicationReport(any(ApplicationId.class))) .thenReturn(appReport); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java index f4987d32061..036fa907fe6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java @@ -26,11 +26,13 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ApplicationTimeout; +import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.LogAggregationStatus; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.proto.YarnProtos.AppTimeoutsMapProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationReportProto; @@ -44,10 +46,11 @@ import com.google.protobuf.TextFormat; -import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Set; @Private @@ -63,7 +66,7 @@ public class ApplicationReportPBImpl extends ApplicationReport { private Token amRmToken = null; private Set applicationTags = null; private Priority priority = null; - private List applicationTimeoutList = null; + private Map applicationTimeouts = null; public ApplicationReportPBImpl() { builder = ApplicationReportProto.newBuilder(); @@ -498,8 +501,8 @@ private void mergeLocalToBuilder() { builder.getPriority())) { builder.setPriority(convertToProtoFormat(this.priority)); } - if (this.applicationTimeoutList != null) { - addLocalApplicationTimeoutToProto(); + if (this.applicationTimeouts != null) { + addApplicationTimeouts(); } } @@ -679,61 +682,38 @@ public void setAmNodeLabelExpression(String amNodeLabelExpression) { } @Override - public List getApplicationTimeouts() { - initLocalApplicationsList(); - return this.applicationTimeoutList; + public Map getApplicationTimeouts() { + initApplicationTimeout(); + return this.applicationTimeouts; } - private void initLocalApplicationsList() { - if (this.applicationTimeoutList != null) { + @Override + public void setApplicationTimeouts( + Map timeouts) { + if (timeouts == null) { + return; + } + initApplicationTimeout(); + this.applicationTimeouts.clear(); + this.applicationTimeouts.putAll(timeouts); + } + + private void initApplicationTimeout() { + if (this.applicationTimeouts != null) { return; } ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; - List list = p.getApplicationTimeoutsList(); - this.applicationTimeoutList = new ArrayList(); - - for (ApplicationTimeoutProto a : list) { - this.applicationTimeoutList.add(convertFromProtoFormat(a)); + List lists = p.getAppTimeoutsList(); + this.applicationTimeouts = + new HashMap(lists.size()); + for (AppTimeoutsMapProto timeoutProto : lists) { + this.applicationTimeouts.put( + ProtoUtils + .convertFromProtoFormat(timeoutProto.getApplicationTimeoutType()), + convertFromProtoFormat(timeoutProto.getApplicationTimeout())); } } - private void addLocalApplicationTimeoutToProto() { - maybeInitBuilder(); - builder.clearApplicationTimeouts(); - if (applicationTimeoutList == null) { - return; - } - Iterable iterable = - new Iterable() { - @Override - public Iterator iterator() { - return new Iterator() { - - private Iterator iter = - applicationTimeoutList.iterator(); - - @Override - public boolean hasNext() { - return iter.hasNext(); - } - - @Override - public ApplicationTimeoutProto next() { - return convertToProtoFormat(iter.next()); - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - - } - }; - - } - }; - builder.addAllApplicationTimeouts(iterable); - } - private ApplicationTimeoutPBImpl convertFromProtoFormat( ApplicationTimeoutProto p) { return new ApplicationTimeoutPBImpl(p); @@ -743,12 +723,45 @@ private ApplicationTimeoutProto convertToProtoFormat(ApplicationTimeout t) { return ((ApplicationTimeoutPBImpl) t).getProto(); } - @Override - public void setApplicationTimeouts(List timeouts) { + private void addApplicationTimeouts() { maybeInitBuilder(); - if (timeouts == null) { - builder.clearApplicationTimeouts(); + builder.clearAppTimeouts(); + if (applicationTimeouts == null) { + return; } - this.applicationTimeoutList = timeouts; + Iterable values = + new Iterable() { + + @Override + public Iterator iterator() { + return new Iterator() { + private Iterator iterator = + applicationTimeouts.keySet().iterator(); + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public AppTimeoutsMapProto next() { + ApplicationTimeoutType key = iterator.next(); + return AppTimeoutsMapProto.newBuilder() + .setApplicationTimeout( + convertToProtoFormat(applicationTimeouts.get(key))) + .setApplicationTimeoutType( + ProtoUtils.convertToProtoFormat(key)) + .build(); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + }; + } + }; + this.builder.addAllAppTimeouts(values); } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 555b49e41d4..7b6da591f0a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -723,10 +723,17 @@ public ApplicationReport createAndGetApplicationReport(String clientUserName, long timeoutInMillis = applicationTimeouts .get(ApplicationTimeoutType.LIFETIME).longValue(); timeout.setExpiryTime(Times.formatISO8601(timeoutInMillis)); - timeout.setRemainingTime( - Math.max((timeoutInMillis - systemClock.getTime()) / 1000, 0)); + if (isAppInCompletedStates()) { + // if application configured with timeout and finished before timeout + // happens then remaining time should not be calculated. + timeout.setRemainingTime(0); + } else { + timeout.setRemainingTime( + Math.max((timeoutInMillis - systemClock.getTime()) / 1000, 0)); + } } - report.setApplicationTimeouts(Collections.singletonList(timeout)); + report.setApplicationTimeouts( + Collections.singletonMap(timeout.getTimeoutType(), timeout)); return report; } finally { this.readLock.unlock(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java index aa4accaddd0..fdc47b9e560 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java @@ -24,7 +24,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Set; @@ -136,15 +135,13 @@ public void testApplicationLifetimeMonitor() throws Exception { GetApplicationReportRequest appRequest = recordFactory.newRecordInstance(GetApplicationReportRequest.class); appRequest.setApplicationId(app2.getApplicationId()); - List applicationTimeoutList = rm.getRMContext() - .getClientRMService().getApplicationReport(appRequest) + Map appTimeouts = rm + .getRMContext().getClientRMService().getApplicationReport(appRequest) .getApplicationReport().getApplicationTimeouts(); - Assert.assertTrue("Application Timeout list are empty.", - !applicationTimeoutList.isEmpty()); - ApplicationTimeout timeout = applicationTimeoutList.iterator().next(); - Assert.assertEquals("Application timeout Type is incorrect.", - ApplicationTimeoutType.LIFETIME.toString(), - timeout.getTimeoutType().toString()); + Assert.assertTrue("Application Timeout are empty.", + !appTimeouts.isEmpty()); + ApplicationTimeout timeout = + appTimeouts.get(ApplicationTimeoutType.LIFETIME); Assert.assertEquals("Application timeout string is incorrect.", formatISO8601, timeout.getExpiryTime()); Assert.assertTrue("Application remaining time is incorrect",