From df3855541af98a3805958f2b5b9db6be705d52ab Mon Sep 17 00:00:00 2001 From: Wangda Tan Date: Thu, 17 Aug 2017 11:30:41 -0700 Subject: [PATCH] YARN-6789. Add Client API to get all supported resource types from RM. (Sunil G via wangda) Change-Id: I366d8db6f6700acd087db5acb7a1be7e41b2b68d --- .../hadoop/mapred/ResourceMgrDelegate.java | 7 + .../hadoop/mapred/TestClientRedirect.java | 9 + .../yarn/api/ApplicationClientProtocol.java | 17 +- .../GetAllResourceTypeInfoRequest.java | 35 ++++ .../GetAllResourceTypeInfoResponse.java | 60 ++++++ .../yarn/api/records/ResourceTypeInfo.java | 196 ++++++++++++++++++ .../yarn/util/resource/ResourceUtils.java | 13 ++ .../proto/applicationclient_protocol.proto | 1 + .../src/main/proto/yarn_protos.proto | 6 + .../src/main/proto/yarn_service_protos.proto | 7 + .../hadoop/yarn/client/api/YarnClient.java | 19 ++ .../yarn/client/api/impl/YarnClientImpl.java | 14 +- ...ApplicationClientProtocolPBClientImpl.java | 18 ++ ...pplicationClientProtocolPBServiceImpl.java | 21 ++ .../GetAllResourceTypeInfoRequestPBImpl.java | 70 +++++++ .../GetAllResourceTypeInfoResponsePBImpl.java | 184 ++++++++++++++++ .../api/records/impl/pb/ResourcePBImpl.java | 12 +- .../impl/pb/ResourceTypeInfoPBImpl.java | 154 ++++++++++++++ .../hadoop/yarn/api/TestPBImplRecords.java | 25 +++ .../hadoop/yarn/api/TestResourcePBImpl.java | 61 ++++++ .../server/MockResourceManagerFacade.java | 8 + .../resourcemanager/ClientRMService.java | 15 +- .../resource/ResourceProfilesManager.java | 8 + .../resource/ResourceProfilesManagerImpl.java | 46 +++- .../resourcemanager/TestClientRMService.java | 46 ++++ .../webapp/TestRMWebServicesApps.java | 2 +- .../DefaultClientRequestInterceptor.java | 8 + .../clientrm/FederationClientInterceptor.java | 8 + .../clientrm/RouterClientRMService.java | 9 + .../PassThroughClientRequestInterceptor.java | 8 + 30 files changed, 1074 insertions(+), 13 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllResourceTypeInfoRequest.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllResourceTypeInfoResponse.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceTypeInfo.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllResourceTypeInfoRequestPBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllResourceTypeInfoResponsePBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceTypeInfoPBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestResourcePBImpl.java diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java index a365f80cc0e..94f741a03e6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java @@ -67,6 +67,7 @@ import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceTypeInfo; import org.apache.hadoop.yarn.api.records.SignalContainerCommand; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; @@ -530,4 +531,10 @@ public class ResourceMgrDelegate extends YarnClient { throws YarnException, IOException { return client.getResourceProfile(profile); } + + @Override + public List getResourceTypeInfo() + throws YarnException, IOException { + return client.getResourceTypeInfo(); + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java index cc50be0a52a..a9b4626581b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java @@ -74,6 +74,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest; import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest; @@ -511,6 +513,13 @@ public class TestClientRedirect { GetResourceProfileRequest request) throws YarnException, IOException { return null; } + + @Override + public GetAllResourceTypeInfoResponse getResourceTypeInfo( + GetAllResourceTypeInfoRequest request) + throws YarnException, IOException { + return null; + } } class HistoryService extends AMService implements HSClientProtocol { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java index 013579f40e6..3112d65c880 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java @@ -67,6 +67,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -625,5 +627,18 @@ public interface ApplicationClientProtocol extends ApplicationBaseProtocol { GetResourceProfileResponse getResourceProfile( GetResourceProfileRequest request) throws YarnException, IOException; - + /** + *

+ * The interface to get the details for a specific resource profile. + *

+ * @param request request to get the details of a resource profile + * @return Response containing the details for a particular resource profile + * @throws YarnException if resource profiles are not enabled on the RM or + * the profile cannot be found + * @throws IOException in case of other errors + */ + @Public + @Unstable + GetAllResourceTypeInfoResponse getResourceTypeInfo( + GetAllResourceTypeInfoRequest request) throws YarnException, IOException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllResourceTypeInfoRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllResourceTypeInfoRequest.java new file mode 100644 index 00000000000..3bda4f54ec5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllResourceTypeInfoRequest.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.util.Records; + +/** + * Request class for getting all the resource profiles from the RM. + */ +@InterfaceAudience.Public +@InterfaceStability.Unstable +public abstract class GetAllResourceTypeInfoRequest { + + public static GetAllResourceTypeInfoRequest newInstance() { + return Records.newRecord(GetAllResourceTypeInfoRequest.class); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllResourceTypeInfoResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllResourceTypeInfoResponse.java new file mode 100644 index 00000000000..b57b96df3fd --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAllResourceTypeInfoResponse.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.api.records.ResourceTypeInfo; +import org.apache.hadoop.yarn.util.Records; + +import java.util.List; + +/** + * Response class for getting all the resource profiles from the RM. + */ +@InterfaceAudience.Public +@InterfaceStability.Unstable +public abstract class GetAllResourceTypeInfoResponse { + + public static GetAllResourceTypeInfoResponse newInstance() { + return Records.newRecord(GetAllResourceTypeInfoResponse.class); + } + + public abstract void setResourceTypeInfo(List resourceTypes); + + public abstract List getResourceTypeInfo(); + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (other == null || !(other instanceof GetAllResourceTypeInfoResponse)) { + return false; + } + return ((GetAllResourceTypeInfoResponse) other).getResourceTypeInfo() + .equals(this.getResourceTypeInfo()); + } + + @Override + public int hashCode() { + return this.getResourceTypeInfo().hashCode(); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceTypeInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceTypeInfo.java new file mode 100644 index 00000000000..6cb470caf1d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceTypeInfo.java @@ -0,0 +1,196 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; +import org.apache.hadoop.yarn.util.Records; + +/** + * Class to encapsulate information about a ResourceType - the name of the + * resource, the units(milli, micro, etc), the type(countable). + */ +public abstract class ResourceTypeInfo implements Comparable { + + /** + * Get the name for the resource. + * + * @return resource name + */ + public abstract String getName(); + + /** + * Set the name for the resource. + * + * @param rName + * name for the resource + */ + public abstract void setName(String rName); + + /** + * Get units for the resource. + * + * @return units for the resource + */ + public abstract String getDefaultUnit(); + + /** + * Set the units for the resource. + * + * @param rUnits + * units for the resource + */ + public abstract void setDefaultUnit(String rUnits); + + /** + * Get the resource type. + * + * @return the resource type + */ + public abstract ResourceTypes getResourceType(); + + /** + * Set the resource type. + * + * @param type + * the resource type + */ + public abstract void setResourceType(ResourceTypes type); + + /** + * Create a new instance of ResourceTypeInfo from another object. + * + * @param other + * the object from which the new object should be created + * @return the new ResourceTypeInfo object + */ + @InterfaceAudience.Public + @InterfaceStability.Unstable + public static ResourceTypeInfo newInstance(ResourceTypeInfo other) { + ResourceTypeInfo resourceType = Records.newRecord(ResourceTypeInfo.class); + copy(other, resourceType); + return resourceType; + } + + /** + * Create a new instance of ResourceTypeInfo from name, units and type. + * + * @param name name of resource type + * @param units units of resource type + * @param type such as countable, etc. + * @return the new ResourceTypeInfo object + */ + @InterfaceAudience.Public + @InterfaceStability.Unstable + public static ResourceTypeInfo newInstance(String name, String units, + ResourceTypes type) { + ResourceTypeInfo resourceType = Records.newRecord(ResourceTypeInfo.class); + resourceType.setName(name); + resourceType.setResourceType(type); + resourceType.setDefaultUnit(units); + return resourceType; + } + + /** + * Create a new instance of ResourceTypeInfo from name, units + * + * @param name name of resource type + * @param units units of resource type + * @return the new ResourceTypeInfo object + */ + @InterfaceAudience.Public + @InterfaceStability.Unstable + public static ResourceTypeInfo newInstance(String name, String units) { + return ResourceTypeInfo.newInstance(name, units, ResourceTypes.COUNTABLE); + } + + /** + * Create a new instance of ResourceTypeInfo from name + * + * @param name name of resource type + * @return the new ResourceTypeInfo object + */ + @InterfaceAudience.Public + @InterfaceStability.Unstable + public static ResourceTypeInfo newInstance(String name) { + return ResourceTypeInfo.newInstance(name, ""); + } + + /** + * Copies the content of the source ResourceTypeInfo object to the + * destination object, overwriting all properties of the destination object. + * + * @param src + * Source ResourceTypeInfo object + * @param dst + * Destination ResourceTypeInfo object + */ + + public static void copy(ResourceTypeInfo src, ResourceTypeInfo dst) { + dst.setName(src.getName()); + dst.setResourceType(src.getResourceType()); + dst.setDefaultUnit(src.getDefaultUnit()); + } + + @Override + public String toString() { + return "name: " + this.getName() + ", units: " + this.getDefaultUnit() + + ", type: " + getResourceType(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (!(obj instanceof ResourceTypeInfo)) { + return false; + } + ResourceTypeInfo r = (ResourceTypeInfo) obj; + return this.getName().equals(r.getName()) + && this.getResourceType().equals(r.getResourceType()) + && this.getDefaultUnit().equals(r.getDefaultUnit()); + } + + @Override + public int hashCode() { + final int prime = 47; + int result = prime + getName().hashCode(); + result = prime * result + getResourceType().hashCode(); + return result; + } + + @Override + public int compareTo(ResourceTypeInfo other) { + int diff = this.getName().compareTo(other.getName()); + if (diff == 0) { + diff = this.getDefaultUnit().compareTo(other.getDefaultUnit()); + if (diff == 0) { + diff = this.getResourceType().compareTo(other.getResourceType()); + } + } + return diff; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java index 06a541477ad..5ed57123f95 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java @@ -548,4 +548,17 @@ public class ResourceUtils { } return ret; } + + /** + * Get default unit by given resource type. + * @param resourceType resourceType + * @return default unit + */ + public static String getDefaultUnit(String resourceType) { + ResourceInformation ri = getResourceTypes().get(resourceType); + if (null != ri) { + return ri.getUnits(); + } + return ""; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto index aac6732f4c4..81adef19335 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto @@ -63,4 +63,5 @@ service ApplicationClientProtocolService { rpc updateApplicationTimeouts (UpdateApplicationTimeoutsRequestProto) returns (UpdateApplicationTimeoutsResponseProto); rpc getResourceProfiles(GetAllResourceProfilesRequestProto) returns (GetAllResourceProfilesResponseProto); rpc getResourceProfile(GetResourceProfileRequestProto) returns (GetResourceProfileResponseProto); + rpc getResourceTypeInfo(GetAllResourceTypeInfoRequestProto) returns (GetAllResourceTypeInfoResponseProto); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index acce41a8354..9933e9ea8e9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -64,6 +64,12 @@ message ResourceInformationProto { optional ResourceTypesProto type = 4; } +message ResourceTypeInfoProto { + required string name = 1; + optional string units = 2; + optional ResourceTypesProto type = 3; +} + message ResourceProto { optional int64 memory = 1; optional int32 virtual_cores = 2; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index 9a9be568a2a..3da4ee7298a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -295,6 +295,13 @@ message GetResourceProfileResponseProto { required ResourceProto resources = 1; } +message GetAllResourceTypeInfoRequestProto { +} + +message GetAllResourceTypeInfoResponseProto { + repeated ResourceTypeInfoProto resource_type_info = 1; +} + ////////////////////////////////////////////////////// /////// client_NM_Protocol /////////////////////////// ////////////////////////////////////////////////////// diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java index 24d28871eb5..4f5b5bcc647 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java @@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.ReservationDefinition; import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceTypeInfo; import org.apache.hadoop.yarn.api.records.SignalContainerCommand; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.YarnApplicationState; @@ -865,6 +866,8 @@ public abstract class YarnClient extends AbstractService { * @throws YarnException if resource profiles are not enabled * @throws IOException in case of other errors */ + @Public + @Unstable public abstract Map getResourceProfiles() throws YarnException, IOException; @@ -878,6 +881,22 @@ public abstract class YarnClient extends AbstractService { * cannot be found * @throws IOException in case of other others */ + @Public + @Unstable public abstract Resource getResourceProfile(String profile) throws YarnException, IOException; + + /** + *

+ * Get available resource types supported by RM. + *

+ * @return list of supported resource types with detailed information + * @throws YarnException if resource profiles are not enabled or the profile + * cannot be found + * @throws IOException in case of other others + */ + @Public + @Unstable + public abstract List getResourceTypeInfo() + throws YarnException, IOException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index b22a6936a61..3c30023f5b2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -44,6 +44,7 @@ import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest; @@ -104,6 +105,7 @@ import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceTypeInfo; import org.apache.hadoop.yarn.api.records.SignalContainerCommand; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.YarnApplicationState; @@ -952,8 +954,16 @@ public class YarnClientImpl extends YarnClient { @Override public Resource getResourceProfile(String profile) throws YarnException, IOException { - GetResourceProfileRequest request = - GetResourceProfileRequest.newInstance(profile); + GetResourceProfileRequest request = GetResourceProfileRequest + .newInstance(profile); return rmClient.getResourceProfile(request).getResource(); } + + @Override + public List getResourceTypeInfo() + throws YarnException, IOException { + GetAllResourceTypeInfoRequest request = + GetAllResourceTypeInfoRequest.newInstance(); + return rmClient.getResourceTypeInfo(request).getResourceTypeInfo(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java index 4bad60d8f90..fd5096a7b37 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java @@ -91,6 +91,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.CancelDelegationTokenRequestPBImpl; @@ -153,6 +155,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationReque import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAllResourceProfilesRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAllResourceProfilesResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAllResourceTypeInfoRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAllResourceTypeInfoResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetResourceProfileRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetResourceProfileResponsePBImpl; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -655,4 +659,18 @@ public class ApplicationClientProtocolPBClientImpl implements ApplicationClientP return null; } } + + @Override + public GetAllResourceTypeInfoResponse getResourceTypeInfo( + GetAllResourceTypeInfoRequest request) throws YarnException, IOException { + YarnServiceProtos.GetAllResourceTypeInfoRequestProto requestProto = + ((GetAllResourceTypeInfoRequestPBImpl) request).getProto(); + try { + return new GetAllResourceTypeInfoResponsePBImpl( + proxy.getResourceTypeInfo(null, requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java index 1c85030e5bb..423287e9105 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java @@ -59,6 +59,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRespo import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.CancelDelegationTokenRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.CancelDelegationTokenResponsePBImpl; @@ -120,6 +121,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationReque import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAllResourceProfilesRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAllResourceProfilesResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAllResourceTypeInfoRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAllResourceTypeInfoResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetResourceProfileRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetResourceProfileResponsePBImpl; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -176,6 +179,8 @@ import org.apache.hadoop.yarn.proto.YarnServiceProtos.UpdateApplicationTimeoutsR import org.apache.hadoop.yarn.proto.YarnServiceProtos.SubmitApplicationRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.SubmitApplicationResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllResourceProfilesResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllResourceTypeInfoRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllResourceTypeInfoResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllResourceProfilesRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileResponseProto; @@ -673,4 +678,20 @@ public class ApplicationClientProtocolPBServiceImpl implements ApplicationClient throw new ServiceException(ie); } } + + @Override + public GetAllResourceTypeInfoResponseProto getResourceTypeInfo( + RpcController controller, GetAllResourceTypeInfoRequestProto proto) + throws ServiceException { + GetAllResourceTypeInfoRequestPBImpl req = new GetAllResourceTypeInfoRequestPBImpl( + proto); + try { + GetAllResourceTypeInfoResponse resp = real.getResourceTypeInfo(req); + return ((GetAllResourceTypeInfoResponsePBImpl) resp).getProto(); + } catch (YarnException ye) { + throw new ServiceException(ye); + } catch (IOException ie) { + throw new ServiceException(ie); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllResourceTypeInfoRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllResourceTypeInfoRequestPBImpl.java new file mode 100644 index 00000000000..b43a3b1a059 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllResourceTypeInfoRequestPBImpl.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllResourceTypeInfoRequestProto; + +/** + * Protobuf implementation class for GetAllResourceTypeInfoRequest. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class GetAllResourceTypeInfoRequestPBImpl + extends GetAllResourceTypeInfoRequest { + + private GetAllResourceTypeInfoRequestProto proto = + GetAllResourceTypeInfoRequestProto.getDefaultInstance(); + private GetAllResourceTypeInfoRequestProto.Builder builder = null; + + private boolean viaProto = false; + + public GetAllResourceTypeInfoRequestPBImpl() { + builder = GetAllResourceTypeInfoRequestProto.newBuilder(); + } + + public GetAllResourceTypeInfoRequestPBImpl( + GetAllResourceTypeInfoRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public GetAllResourceTypeInfoRequestProto getProto() { + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllResourceTypeInfoResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllResourceTypeInfoResponsePBImpl.java new file mode 100644 index 00000000000..28decebcabf --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAllResourceTypeInfoResponsePBImpl.java @@ -0,0 +1,184 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse; +import org.apache.hadoop.yarn.api.records.ResourceTypeInfo; +import org.apache.hadoop.yarn.api.records.impl.pb.ResourceTypeInfoPBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.ResourceTypeInfoProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllResourceTypeInfoResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllResourceTypeInfoResponseProtoOrBuilder; + +import com.google.protobuf.TextFormat; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +/** + * Protobuf implementation class for the GetAllResourceTypeInfoResponse. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class GetAllResourceTypeInfoResponsePBImpl + extends + GetAllResourceTypeInfoResponse { + + private GetAllResourceTypeInfoResponseProto proto = GetAllResourceTypeInfoResponseProto + .getDefaultInstance(); + private GetAllResourceTypeInfoResponseProto.Builder builder = null; + private boolean viaProto = false; + + private List resourceTypeInfo; + + public GetAllResourceTypeInfoResponsePBImpl() { + builder = GetAllResourceTypeInfoResponseProto.newBuilder(); + } + + public GetAllResourceTypeInfoResponsePBImpl( + GetAllResourceTypeInfoResponseProto proto) { + this.proto = proto; + viaProto = true; + } + + public GetAllResourceTypeInfoResponseProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public void setResourceTypeInfo(List resourceTypes) { + if (resourceTypeInfo == null) { + builder.clearResourceTypeInfo(); + } + this.resourceTypeInfo = resourceTypes; + } + + @Override + public List getResourceTypeInfo() { + initResourceTypeInfosList(); + return this.resourceTypeInfo; + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + private void mergeLocalToBuilder() { + if (this.resourceTypeInfo != null) { + addResourceTypeInfosToProto(); + } + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = GetAllResourceTypeInfoResponseProto.newBuilder(proto); + } + viaProto = false; + } + + // Once this is called. containerList will never be null - until a getProto + // is called. + private void initResourceTypeInfosList() { + if (this.resourceTypeInfo != null) { + return; + } + GetAllResourceTypeInfoResponseProtoOrBuilder p = viaProto ? proto : builder; + List list = p.getResourceTypeInfoList(); + resourceTypeInfo = new ArrayList(); + + for (ResourceTypeInfoProto a : list) { + resourceTypeInfo.add(convertFromProtoFormat(a)); + } + } + + private void addResourceTypeInfosToProto() { + maybeInitBuilder(); + builder.clearResourceTypeInfo(); + if (resourceTypeInfo == null) { + return; + } + Iterable iterable = new Iterable() { + @Override + public Iterator iterator() { + return new Iterator() { + + Iterator iter = resourceTypeInfo.iterator(); + + @Override + public boolean hasNext() { + return iter.hasNext(); + } + + @Override + public ResourceTypeInfoProto next() { + return convertToProtoFormat(iter.next()); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + + } + }; + + } + }; + builder.addAllResourceTypeInfo(iterable); + } + + private ResourceTypeInfoPBImpl convertFromProtoFormat( + ResourceTypeInfoProto p) { + return new ResourceTypeInfoPBImpl(p); + } + + private ResourceTypeInfoProto convertToProtoFormat(ResourceTypeInfo t) { + return ((ResourceTypeInfoPBImpl) t).getProto(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java index cbb040a7e87..ff22dd019c3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java @@ -63,6 +63,7 @@ public class ResourcePBImpl extends BaseResource { public ResourcePBImpl() { builder = ResourceProto.newBuilder(); + initResources(); } public ResourcePBImpl(ResourceProto proto) { @@ -94,7 +95,6 @@ public class ResourcePBImpl extends BaseResource { @Override public long getMemorySize() { // memory should always be present - initResources(); ResourceInformation ri = resources[MandatoryResources.MEMORY.getId()]; if (ri.getUnits().isEmpty()) { @@ -119,7 +119,6 @@ public class ResourcePBImpl extends BaseResource { @Override public int getVirtualCores() { // vcores should always be present - initResources(); return (int) resources[MandatoryResources.VCORES.getId()].getValue(); } @@ -140,7 +139,11 @@ public class ResourcePBImpl extends BaseResource { ResourceTypes type = entry.hasType() ? ProtoUtils.convertFromProtoFormat(entry.getType()) : ResourceTypes.COUNTABLE; - String units = entry.hasUnits() ? entry.getUnits() : ""; + + // When unit not specified in proto, use the default unit. + String units = + entry.hasUnits() ? entry.getUnits() : ResourceUtils.getDefaultUnit( + entry.getKey()); long value = entry.hasValue() ? entry.getValue() : 0L; ResourceInformation ri = ResourceInformation .newInstance(entry.getKey(), units, value, type, 0L, Long.MAX_VALUE); @@ -185,21 +188,18 @@ public class ResourcePBImpl extends BaseResource { @Override public ResourceInformation[] getResources() { - initResources(); return super.getResources(); } @Override public ResourceInformation getResourceInformation(String resource) throws ResourceNotFoundException { - initResources(); return super.getResourceInformation(resource); } @Override public long getResourceValue(String resource) throws ResourceNotFoundException { - initResources(); return super.getResourceValue(resource); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceTypeInfoPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceTypeInfoPBImpl.java new file mode 100644 index 00000000000..0454b961684 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceTypeInfoPBImpl.java @@ -0,0 +1,154 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; +import org.apache.hadoop.yarn.api.records.ResourceTypeInfo; +import org.apache.hadoop.yarn.proto.YarnProtos; +import org.apache.hadoop.yarn.proto.YarnProtos.ResourceTypeInfoProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ResourceTypesProto; + +import com.google.common.base.Preconditions; + +/** + * {@code ResourceTypeInfoPBImpl} which implements the + * {@link ResourceTypeInfo} class which represents different resource types + * supported in YARN. + */ +@Private +@Unstable +public class ResourceTypeInfoPBImpl extends ResourceTypeInfo { + + ResourceTypeInfoProto proto = ResourceTypeInfoProto.getDefaultInstance(); + ResourceTypeInfoProto.Builder builder = null; + boolean viaProto = false; + + private String name = null; + private String defaultUnit = null; + private ResourceTypes resourceTypes = null; + + public ResourceTypeInfoPBImpl() { + builder = ResourceTypeInfoProto.newBuilder(); + } + + public ResourceTypeInfoPBImpl(ResourceTypeInfoProto proto) { + this.proto = proto; + viaProto = true; + } + + public ResourceTypeInfoProto getProto() { + mergeLocalToProto(); + return proto; + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void mergeLocalToBuilder() { + if (this.name != null) { + builder.setName(this.name); + } + if (this.defaultUnit != null) { + builder.setUnits(this.defaultUnit); + } + if (this.resourceTypes != null) { + builder.setType(convertToProtoFormat(this.resourceTypes)); + } + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = YarnProtos.ResourceTypeInfoProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public String getName() { + if (this.name != null) { + return this.name; + } + + YarnProtos.ResourceTypeInfoProtoOrBuilder p = viaProto ? proto : builder; + return p.getName(); + } + + @Override + public void setName(String rName) { + maybeInitBuilder(); + if (rName == null) { + builder.clearName(); + } + this.name = rName; + } + + @Override + public String getDefaultUnit() { + if (this.defaultUnit != null) { + return this.defaultUnit; + } + + YarnProtos.ResourceTypeInfoProtoOrBuilder p = viaProto ? proto : builder; + return p.getUnits(); + } + + @Override + public void setDefaultUnit(String rUnits) { + maybeInitBuilder(); + if (rUnits == null) { + builder.clearUnits(); + } + this.defaultUnit = rUnits; + } + + @Override + public ResourceTypes getResourceType() { + if (this.resourceTypes != null) { + return this.resourceTypes; + } + + YarnProtos.ResourceTypeInfoProtoOrBuilder p = viaProto ? proto : builder; + return convertFromProtoFormat(p.getType()); + } + + @Override + public void setResourceType(ResourceTypes type) { + maybeInitBuilder(); + if (type == null) { + builder.clearType(); + } + this.resourceTypes = type; + } + + public static ResourceTypesProto convertToProtoFormat(ResourceTypes e) { + return ResourceTypesProto.valueOf(e.name()); + } + + public static ResourceTypes convertFromProtoFormat(ResourceTypesProto e) { + return ResourceTypes.valueOf(e.name()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java index 3f84412b14a..11e53db0014 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java @@ -29,6 +29,8 @@ import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; import org.apache.hadoop.yarn.api.protocolrecords.CommitResponse; import org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateRequest; import org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest; import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse; import org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerRequest; @@ -43,6 +45,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.CancelDelegationTokenR import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.FinishApplicationMasterRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.FinishApplicationMasterResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAllResourceProfilesResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAllResourceTypeInfoRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAllResourceTypeInfoResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationAttemptReportRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationAttemptReportResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationAttemptsRequestPBImpl; @@ -148,6 +152,7 @@ import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; import org.apache.hadoop.yarn.api.records.ResourceOption; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.ResourceTypeInfo; import org.apache.hadoop.yarn.api.records.ResourceUtilization; import org.apache.hadoop.yarn.api.records.SerializedException; import org.apache.hadoop.yarn.api.records.StrictPreemptionContract; @@ -187,12 +192,14 @@ import org.apache.hadoop.yarn.api.records.impl.pb.ResourceBlacklistRequestPBImpl import org.apache.hadoop.yarn.api.records.impl.pb.ResourceOptionPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ResourceRequestPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.ResourceTypeInfoPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.SerializedExceptionPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.StrictPreemptionContractPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.TokenPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.URLPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.UpdateContainerRequestPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.YarnClusterMetricsPBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptReportProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto; @@ -1195,4 +1202,22 @@ public class TestPBImplRecords extends BasePBImplRecordsTest { validatePBImplRecord(ProfileCapabilityPBImpl.class, ProfileCapabilityProto.class); } + + @Test + public void testResourceTypesInfoPBImpl() throws Exception { + validatePBImplRecord(ResourceTypeInfoPBImpl.class, + YarnProtos.ResourceTypeInfoProto.class); + } + + @Test + public void testGetAllResourceTypesInfoRequestPBImpl() throws Exception { + validatePBImplRecord(GetAllResourceTypeInfoRequestPBImpl.class, + YarnServiceProtos.GetAllResourceTypeInfoRequestProto.class); + } + + @Test + public void testGetAllResourceTypesInfoResponsePBImpl() throws Exception { + validatePBImplRecord(GetAllResourceTypeInfoResponsePBImpl.class, + YarnServiceProtos.GetAllResourceTypeInfoResponseProto.class); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestResourcePBImpl.java new file mode 100644 index 00000000000..569a7b74f8b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestResourcePBImpl.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api; + +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos; +import org.junit.Assert; +import org.junit.Test; + +/** + * Test class to handle various proto related tests for resources. + */ +public class TestResourcePBImpl { + @Test + public void testEmptyResourcePBInit() throws Exception { + Resource res = new ResourcePBImpl(); + // Assert to check it sets resource value and unit to default. + Assert.assertEquals(0, res.getMemorySize()); + Assert.assertEquals(ResourceInformation.MEMORY_MB.getUnits(), + res.getResourceInformation(ResourceInformation.MEMORY_MB.getName()) + .getUnits()); + Assert.assertEquals(ResourceInformation.VCORES.getUnits(), + res.getResourceInformation(ResourceInformation.VCORES.getName()) + .getUnits()); + } + + @Test + public void testResourcePBInitFromOldPB() throws Exception { + YarnProtos.ResourceProto proto = + YarnProtos.ResourceProto.newBuilder().setMemory(1024).setVirtualCores(3) + .build(); + // Assert to check it sets resource value and unit to default. + Resource res = new ResourcePBImpl(proto); + Assert.assertEquals(1024, res.getMemorySize()); + Assert.assertEquals(3, res.getVirtualCores()); + Assert.assertEquals(ResourceInformation.MEMORY_MB.getUnits(), + res.getResourceInformation(ResourceInformation.MEMORY_MB.getName()) + .getUnits()); + Assert.assertEquals(ResourceInformation.VCORES.getUnits(), + res.getResourceInformation(ResourceInformation.VCORES.getName()) + .getUnits()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java index a5831be20b1..769296b74b2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java @@ -43,6 +43,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest; @@ -868,4 +870,10 @@ public class MockResourceManagerFacade implements ApplicationClientProtocol, GetResourceProfileRequest request) throws YarnException, IOException { return null; } + + @Override + public GetAllResourceTypeInfoResponse getResourceTypeInfo( + GetAllResourceTypeInfoRequest request) throws YarnException, IOException { + return null; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index a49cf1e4134..0d65838f382 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -114,6 +114,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsReque import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetResourceProfileResponse; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; @@ -1794,8 +1796,7 @@ public class ClientRMService extends AbstractService implements } GetResourceProfileResponse response = GetResourceProfileResponse.newInstance(); - response.setResource( - resourceProfilesManager.getProfile(request.getProfileName())); + response.setResource(profiles.get(request.getProfileName())); return response; } @@ -1809,4 +1810,14 @@ public class ClientRMService extends AbstractService implements } return resourceProfilesManager.getResourceProfiles(); } + + @Override + public GetAllResourceTypeInfoResponse getResourceTypeInfo( + GetAllResourceTypeInfoRequest request) throws YarnException, IOException { + GetAllResourceTypeInfoResponse response = + GetAllResourceTypeInfoResponse.newInstance(); + response.setResourceTypeInfo( + resourceProfilesManager.getAllResourceTypeInfo()); + return response; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManager.java index c330e251281..896312178ee 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManager.java @@ -22,8 +22,10 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceTypeInfo; import java.io.IOException; +import java.util.List; import java.util.Map; /** @@ -77,4 +79,10 @@ public interface ResourceProfilesManager { * @return resource object which is maximum */ Resource getMaximumProfile(); + + /** + * List of ResourceTypeInfo objects which carry all resources supported by RM. + * @return list of ResourceTypeInfo objects + */ + List getAllResourceTypeInfo(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java index 42d38b4518c..ff4e3e4ec30 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java @@ -24,6 +24,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.api.records.ResourceTypeInfo; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.resource.Resources; @@ -32,11 +33,15 @@ import org.codehaus.jackson.map.ObjectMapper; import java.io.File; import java.io.IOException; import java.net.URL; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.ReentrantReadWriteLock; public class ResourceProfilesManagerImpl implements ResourceProfilesManager { @@ -44,6 +49,8 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager { LogFactory.getLog(ResourceProfilesManagerImpl.class); private final Map profiles = new ConcurrentHashMap<>(); + private List resourceTypeInfo = + new ArrayList(); private Configuration conf; private static final String MEMORY = ResourceInformation.MEMORY_MB.getName(); @@ -53,13 +60,41 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager { public static final String MINIMUM_PROFILE = "minimum"; public static final String MAXIMUM_PROFILE = "maximum"; + protected final ReentrantReadWriteLock.ReadLock readLock; + protected final ReentrantReadWriteLock.WriteLock writeLock; + private static final String[] MANDATORY_PROFILES = { DEFAULT_PROFILE, MINIMUM_PROFILE, MAXIMUM_PROFILE }; - @Override + public ResourceProfilesManagerImpl() { + ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + readLock = lock.readLock(); + writeLock = lock.writeLock(); + } + public void init(Configuration config) throws IOException { conf = config; loadProfiles(); + + // Load resource types, this should be done even if resource profile is + // disabled, since we have mandatory resource types like vcores/memory. + loadResourceTypes(); + } + + private void loadResourceTypes() { + // Add all resource types + try { + writeLock.lock(); + Collection resourcesInfo = ResourceUtils + .getResourceTypes().values(); + for (ResourceInformation resourceInfo : resourcesInfo) { + resourceTypeInfo + .add(ResourceTypeInfo.newInstance(resourceInfo.getName(), + resourceInfo.getUnits(), resourceInfo.getResourceType())); + } + } finally { + writeLock.unlock(); + } } private void loadProfiles() throws IOException { @@ -185,4 +220,13 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager { Long.valueOf(value.substring(0, value.length() - units.length())); return ResourceInformation.newInstance(name, units, resourceValue); } + + public List getAllResourceTypeInfo() { + try { + readLock.lock(); + return Collections.unmodifiableList(resourceTypeInfo); + } finally { + readLock.unlock(); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index ea733a4ce4a..49718a2af72 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -56,6 +56,8 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.MockApps; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest; @@ -116,6 +118,7 @@ import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.api.records.ReservationRequest; import org.apache.hadoop.yarn.api.records.ReservationRequests; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -2061,4 +2064,47 @@ public class TestClientRMService { rpc.stopProxy(client, conf); new File(excludeFile).delete(); } + + @Test + public void testGetResourceTypesInfoWhenResourceProfileDisabled() + throws Exception { + YarnConfiguration conf = new YarnConfiguration(); + MockRM rm = new MockRM(conf) { + protected ClientRMService createClientRMService() { + return new ClientRMService(this.rmContext, scheduler, + this.rmAppManager, this.applicationACLsManager, this.queueACLsManager, + this.getRMContext().getRMDelegationTokenSecretManager()); + } + }; + rm.start(); + + YarnRPC rpc = YarnRPC.create(conf); + InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress(); + LOG.info("Connecting to ResourceManager at " + rmAddress); + ApplicationClientProtocol client = + (ApplicationClientProtocol) rpc + .getProxy(ApplicationClientProtocol.class, rmAddress, conf); + + // Make call + GetAllResourceTypeInfoRequest request = + GetAllResourceTypeInfoRequest.newInstance(); + GetAllResourceTypeInfoResponse response = client.getResourceTypeInfo(request); + + Assert.assertEquals(2, response.getResourceTypeInfo().size()); + + // Check memory + Assert.assertEquals(ResourceInformation.MEMORY_MB.getName(), + response.getResourceTypeInfo().get(0).getName()); + Assert.assertEquals(ResourceInformation.MEMORY_MB.getUnits(), + response.getResourceTypeInfo().get(0).getDefaultUnit()); + + // Check vcores + Assert.assertEquals(ResourceInformation.VCORES.getName(), + response.getResourceTypeInfo().get(1).getName()); + Assert.assertEquals(ResourceInformation.VCORES.getUnits(), + response.getResourceTypeInfo().get(1).getDefaultUnit()); + + rm.stop(); + rpc.stopProxy(client, conf); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 1cbdec3a225..07f74a358e7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1528,7 +1528,7 @@ public class TestRMWebServicesApps extends JerseyTestBase { public void verifyAppInfo(JSONObject info, RMApp app, boolean hasResourceReqs) throws JSONException, Exception { - int expectedNumberOfElements = 36 + (hasResourceReqs ? 2 : 0); + int expectedNumberOfElements = 38 + (hasResourceReqs ? 2 : 0); String appNodeLabelExpression = null; String amNodeLabelExpression = null; if (app.getApplicationSubmissionContext() diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/DefaultClientRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/DefaultClientRequestInterceptor.java index d2271dbc84d..5ce4803f603 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/DefaultClientRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/DefaultClientRequestInterceptor.java @@ -29,6 +29,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest; import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest; @@ -319,6 +321,12 @@ public class DefaultClientRequestInterceptor return clientRMProxy.getResourceProfile(request); } + @Override + public GetAllResourceTypeInfoResponse getResourceTypeInfo( + GetAllResourceTypeInfoRequest request) throws YarnException, IOException { + return clientRMProxy.getResourceTypeInfo(request); + } + @VisibleForTesting public void setRMClient(ApplicationClientProtocol clientRM) { this.clientRMProxy = clientRM; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java index 09698851870..b8f8a9fd841 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java @@ -34,6 +34,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest; import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest; @@ -724,4 +726,10 @@ public class FederationClientInterceptor GetResourceProfileRequest request) throws YarnException, IOException { throw new NotImplementedException(); } + + @Override + public GetAllResourceTypeInfoResponse getResourceTypeInfo( + GetAllResourceTypeInfoRequest request) throws YarnException, IOException { + throw new NotImplementedException(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java index ae1b73e7d3c..73cc18558d1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java @@ -40,6 +40,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest; import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest; @@ -421,6 +423,13 @@ public class RouterClientRMService extends AbstractService return pipeline.getRootInterceptor().getResourceProfile(request); } + @Override + public GetAllResourceTypeInfoResponse getResourceTypeInfo( + GetAllResourceTypeInfoRequest request) throws YarnException, IOException { + RequestInterceptorChainWrapper pipeline = getInterceptorChain(); + return pipeline.getRootInterceptor().getResourceTypeInfo(request); + } + private RequestInterceptorChainWrapper getInterceptorChain() throws IOException { String user = UserGroupInformation.getCurrentUser().getUserName(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/PassThroughClientRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/PassThroughClientRequestInterceptor.java index 2cac6852ef6..cb1b529d8eb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/PassThroughClientRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/PassThroughClientRequestInterceptor.java @@ -26,6 +26,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest; import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceProfilesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAllResourceTypeInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest; @@ -280,4 +282,10 @@ public class PassThroughClientRequestInterceptor GetResourceProfileRequest request) throws YarnException, IOException { return getNextInterceptor().getResourceProfile(request); } + + @Override + public GetAllResourceTypeInfoResponse getResourceTypeInfo( + GetAllResourceTypeInfoRequest request) throws YarnException, IOException { + return getNextInterceptor().getResourceTypeInfo(request); + } }