Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-7043. Cleanup ResourceProfileManager. (wangda) 

Change-Id: I463356f37bf1f6a3f1fc3c594c79916e8c0ab913
This commit is contained in:
Wangda Tan 2017-08-21 17:20:06 -07:00
parent 758b771943
commit 39240b61a1
17 changed files with 192 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceProcessor.java
View File

@ -52,11 +52,13 @@ public interface ApplicationMasterServiceProcessor {
* @param request Register Request.
* @param response Register Response.
* @throws IOException IOException.
* @throws YarnException in critical situation where invalid
* profiles/resources are added.
*/
void registerApplicationMaster(
ApplicationAttemptId applicationAttemptId,
void registerApplicationMaster(ApplicationAttemptId applicationAttemptId,
RegisterApplicationMasterRequest request,
RegisterApplicationMasterResponse response) throws IOException;
RegisterApplicationMasterResponse response)
throws IOException, YarnException;
/**
* Allocate call.

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java
View File

@ -81,6 +81,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YARNFeatureNotEnabledException;
/**
* <p>The protocol between clients and the <code>ResourceManager</code>
@ -604,7 +605,8 @@ public interface ApplicationClientProtocol extends ApplicationBaseProtocol {
* @param request request to get all the resource profiles
* @return Response containing a map of the profile name to Resource
* capabilities
* @throws YarnException if resource profiles are not enabled on the RM
* @throws YARNFeatureNotEnabledException if resource-profile is disabled
* @throws YarnException if any error happens inside YARN
* @throws IOException in case of other errors
*/
@Public
@ -618,8 +620,8 @@ public interface ApplicationClientProtocol extends ApplicationBaseProtocol {
* </p>
* @param request request to get the details of a resource profile
* @return Response containing the details for a particular resource profile
* @throws YarnException if resource profiles are not enabled on the RM or
* the profile cannot be found
* @throws YARNFeatureNotEnabledException if resource-profile is disabled
* @throws YarnException if any error happens inside YARN
* @throws IOException in case of other errors
*/
@Public
@ -633,8 +635,7 @@ public interface ApplicationClientProtocol extends ApplicationBaseProtocol {
* </p>
* @param request request to get the details of a resource profile
* @return Response containing the details for a particular resource profile
* @throws YarnException if resource profiles are not enabled on the RM or
* the profile cannot be found
* @throws YarnException if any error happens inside YARN
* @throws IOException in case of other errors
*/
@Public

45
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/YARNFeatureNotEnabledException.java Normal file
View File

@ -0,0 +1,45 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.exceptions;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
/**
* This exception is thrown when a feature is being used which is not enabled
* yet.
*/
@InterfaceAudience.Public
@InterfaceStability.Unstable
public class YARNFeatureNotEnabledException extends YarnException {
private static final long serialVersionUID = 898023752676L;
public YARNFeatureNotEnabledException(Throwable cause) {
super(cause);
}
public YARNFeatureNotEnabledException(String message) {
super(message);
}
public YARNFeatureNotEnabledException(String message, Throwable cause) {
super(message, cause);
}
}

21
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
View File

@ -28,6 +28,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
import org.apache.hadoop.yarn.conf.ConfigurationProvider;
import org.apache.hadoop.yarn.conf.ConfigurationProviderFactory;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@ -37,9 +38,12 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
@ -561,4 +565,21 @@ public class ResourceUtils {
}
return "";
}
/**
* Get all resource types information from known resource types.
* @return List of ResourceTypeInfo
*/
public static List<ResourceTypeInfo> getResourcesTypeInfo() {
List<ResourceTypeInfo> array = new ArrayList<>();
// Add all resource types
Collection<ResourceInformation> resourcesInfo =
ResourceUtils.getResourceTypes().values();
for (ResourceInformation resourceInfo : resourcesInfo) {
array.add(ResourceTypeInfo
.newInstance(resourceInfo.getName(), resourceInfo.getUnits(),
resourceInfo.getResourceType()));
}
return array;
}
}

13
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
View File

@ -72,6 +72,7 @@ import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
import org.apache.hadoop.yarn.exceptions.ApplicationIdNotProvidedException;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
import org.apache.hadoop.yarn.exceptions.YARNFeatureNotEnabledException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
@ -863,7 +864,8 @@ public abstract class YarnClient extends AbstractService {
* Get the resource profiles available in the RM.
* </p>
* @return a Map of the resource profile names to their capabilities
* @throws YarnException if resource profiles are not enabled
* @throws YARNFeatureNotEnabledException if resource-profile is disabled
* @throws YarnException if any error happens inside YARN
* @throws IOException in case of other errors
*/
@Public
@ -876,9 +878,9 @@ public abstract class YarnClient extends AbstractService {
* Get the details of a specific resource profile from the RM.
* </p>
* @param profile the profile name
* @return the capabilities of the resource profile
* @throws YarnException if resource profiles are not enabled or the profile
* cannot be found
* @return resource profile name with its capabilities
* @throws YARNFeatureNotEnabledException if resource-profile is disabled
* @throws YarnException if any error happens inside YARN
* @throws IOException in case of other others
*/
@Public
@ -891,8 +893,7 @@ public abstract class YarnClient extends AbstractService {
* Get available resource types supported by RM.
* </p>
* @return list of supported resource types with detailed information
* @throws YarnException if resource profiles are not enabled or the profile
* cannot be found
* @throws YarnException if any issue happens inside YARN
* @throws IOException in case of other others
*/
@Public

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
View File

@ -528,10 +528,12 @@ public class DominantResourceCalculator extends ResourceCalculator {
long value;
if (stepFactorValue != 0) {
value = roundUp
? roundUp((long) Math.ceil(rValue * by), stepFactorValue)
? roundUp((long) Math.ceil((float) (rValue * by)), stepFactorValue)
: roundDown((long) (rValue * by), stepFactorValue);
} else {
value = roundUp ? (long) Math.ceil(rValue * by) : (long) (rValue * by);
value = roundUp
? (long) Math.ceil((float) (rValue * by))
: (long) (rValue * by);
}
tmp.setValue(value);
}

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
View File

@ -431,6 +431,7 @@ public class TestPBImplRecords extends BasePBImplRecordsTest {
generateByNewInstance(ApplicationTimeout.class);
generateByNewInstance(QueueConfigurations.class);
generateByNewInstance(CollectorInfo.class);
generateByNewInstance(ResourceTypeInfo.class);
}
@Test

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AMSProcessingChain.java
View File

@ -82,7 +82,7 @@ class AMSProcessingChain implements ApplicationMasterServiceProcessor {
public void registerApplicationMaster(
ApplicationAttemptId applicationAttemptId,
RegisterApplicationMasterRequest request,
RegisterApplicationMasterResponse resp) throws IOException {
RegisterApplicationMasterResponse resp) throws IOException, YarnException {
this.head.registerApplicationMaster(applicationAttemptId, request, resp);
}

26
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
View File

@ -143,7 +143,6 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
import org.apache.hadoop.yarn.exceptions.ResourceProfilesNotEnabledException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@ -183,6 +182,7 @@ import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.util.UTCClock;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
@ -1782,42 +1782,26 @@ public class ClientRMService extends AbstractService implements
GetAllResourceProfilesRequest request) throws YarnException, IOException {
GetAllResourceProfilesResponse response =
GetAllResourceProfilesResponse.newInstance();
response.setResourceProfiles(getResourceProfiles());
response.setResourceProfiles(resourceProfilesManager.getResourceProfiles());
return response;
}
@Override
public GetResourceProfileResponse getResourceProfile(
GetResourceProfileRequest request) throws YarnException, IOException {
Map<String, Resource> profiles = getResourceProfiles();
if (!profiles.containsKey(request.getProfileName())) {
throw new YarnException(
"Resource profile '" + request.getProfileName() + "' not found");
}
GetResourceProfileResponse response =
GetResourceProfileResponse.newInstance();
response.setResource(profiles.get(request.getProfileName()));
response.setResource(
resourceProfilesManager.getProfile(request.getProfileName()));
return response;
}
private Map<String, Resource> getResourceProfiles() throws YarnException {
boolean resourceProfilesEnabled = getConfig()
.getBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED,
YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED);
if (!resourceProfilesEnabled) {
throw new ResourceProfilesNotEnabledException(
"Resource profiles are not enabled");
}
return resourceProfilesManager.getResourceProfiles();
}
@Override
public GetAllResourceTypeInfoResponse getResourceTypeInfo(
GetAllResourceTypeInfoRequest request) throws YarnException, IOException {
GetAllResourceTypeInfoResponse response =
GetAllResourceTypeInfoResponse.newInstance();
response.setResourceTypeInfo(
resourceProfilesManager.getAllResourceTypeInfo());
response.setResourceTypeInfo(ResourceUtils.getResourcesTypeInfo());
return response;
}
}

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
View File

@ -114,7 +114,8 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor {
public void registerApplicationMaster(
ApplicationAttemptId applicationAttemptId,
RegisterApplicationMasterRequest request,
RegisterApplicationMasterResponse response) throws IOException {
RegisterApplicationMasterResponse response)
throws IOException, YarnException {
RMApp app = getRmContext().getRMApps().get(
applicationAttemptId.getApplicationId());
@ -174,10 +175,11 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor {
response.setSchedulerResourceTypes(getScheduler()
.getSchedulingResourceTypes());
if (getRmContext().getYarnConfiguration().getBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED,
YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED)) {
response
.setResourceProfiles(resourceProfilesManager.getResourceProfiles());
if (getRmContext().getYarnConfiguration().getBoolean(
YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED,
YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED)) {
response.setResourceProfiles(
resourceProfilesManager.getResourceProfiles());
}
}

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
View File

@ -127,7 +127,8 @@ public class OpportunisticContainerAllocatorAMService
public void registerApplicationMaster(
ApplicationAttemptId applicationAttemptId,
RegisterApplicationMasterRequest request,
RegisterApplicationMasterResponse response) throws IOException {
RegisterApplicationMasterResponse response)
throws IOException, YarnException {
SchedulerApplicationAttempt appAttempt = ((AbstractYarnScheduler)
getScheduler()).getApplicationAttempt(applicationAttemptId);
if (appAttempt.getOpportunisticContainerContext() == null) {

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
View File

@ -628,20 +628,6 @@ public class RMServerUtils {
}
}
public static void convertProfileToResourceCapability(
List<ResourceRequest> asks, Configuration conf,
ResourceProfilesManager resourceProfilesManager) throws YarnException {
boolean profilesEnabled =
conf.getBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED,
YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED);
if (!profilesEnabled) {
return;
}
for (ResourceRequest req : asks) {
convertProfileToResourceCapability(req, conf, resourceProfilesManager);
}
}
public static void convertProfileToResourceCapability(ResourceRequest ask,
Configuration conf, ResourceProfilesManager resourceProfilesManager)
throws YarnException {

32
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManager.java
View File

@ -18,22 +18,18 @@
package org.apache.hadoop.yarn.server.resourcemanager.resource;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
import org.apache.hadoop.yarn.exceptions.YARNFeatureNotEnabledException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import java.io.IOException;
import java.util.List;
import java.util.Map;
/**
* Interface for the resource profiles manager. Provides an interface to get
* the list of available profiles and some helper functions.
*/
@Public
@Unstable
public interface ResourceProfilesManager {
/**
@ -47,14 +43,19 @@ public interface ResourceProfilesManager {
* Get the resource capability associated with given profile name.
* @param profile name of resource profile
* @return resource capability for given profile
*
* @throws YarnException when any invalid profile name or feature is disabled
*/
Resource getProfile(String profile);
Resource getProfile(String profile) throws YarnException;
/**
* Get all supported resource profiles.
* @return a map of resource objects associated with each profile
*
* @throws YARNFeatureNotEnabledException when feature is disabled
*/
Map<String, Resource> getResourceProfiles();
Map<String, Resource> getResourceProfiles() throws
YARNFeatureNotEnabledException;
/**
* Reload profiles based on updated configuration.
@ -65,24 +66,21 @@ public interface ResourceProfilesManager {
/**
* Get default supported resource profile.
* @return resource object which is default
* @throws YarnException when any invalid profile name or feature is disabled
*/
Resource getDefaultProfile();
Resource getDefaultProfile() throws YarnException;
/**
* Get minimum supported resource profile.
* @return resource object which is minimum
* @throws YarnException when any invalid profile name or feature is disabled
*/
Resource getMinimumProfile();
Resource getMinimumProfile() throws YarnException;
/**
* Get maximum supported resource profile.
* @return resource object which is maximum
* @throws YarnException when any invalid profile name or feature is disabled
*/
Resource getMaximumProfile();
/**
* List of ResourceTypeInfo objects which carry all resources supported by RM.
* @return list of ResourceTypeInfo objects
*/
List<ResourceTypeInfo> getAllResourceTypeInfo();
Resource getMaximumProfile() throws YarnException;
}

63
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java
View File

@ -26,6 +26,8 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YARNFeatureNotEnabledException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.codehaus.jackson.map.ObjectMapper;
@ -52,6 +54,7 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager {
private List<ResourceTypeInfo> resourceTypeInfo =
new ArrayList<ResourceTypeInfo>();
private Configuration conf;
private boolean profileEnabled = false;
private static final String MEMORY = ResourceInformation.MEMORY_MB.getName();
private static final String VCORES = ResourceInformation.VCORES.getName();
@ -65,6 +68,11 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager {
private static final String[] MANDATORY_PROFILES =
{ DEFAULT_PROFILE, MINIMUM_PROFILE, MAXIMUM_PROFILE };
private static final String FEATURE_NOT_ENABLED_MSG =
"Resource profile is not enabled, please "
+ "enable resource profile feature before using its functions."
+ " (by setting " + YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED
+ " to true)";
public ResourceProfilesManagerImpl() {
ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
@ -98,10 +106,10 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager {
}
private void loadProfiles() throws IOException {
boolean profilesEnabled =
profileEnabled =
conf.getBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED,
YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED);
if (!profilesEnabled) {
if (!profileEnabled) {
return;
}
String sourceFile =
@ -131,14 +139,16 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager {
if (entry.getValue() instanceof Map) {
Map profileInfo = (Map) entry.getValue();
// ensure memory and vcores are specified
if (!profileInfo.containsKey(MEMORY) || !profileInfo.containsKey(VCORES)) {
if (!profileInfo.containsKey(MEMORY)
|| !profileInfo.containsKey(VCORES)) {
throw new IOException(
"Illegal resource profile definition; profile '" + profileName
+ "' must contain '" + MEMORY + "' and '" + VCORES + "'");
}
Resource resource = parseResource(profileInfo);
profiles.put(profileName, resource);
LOG.info("Added profile '" + profileName + "' with resources " + resource);
LOG.info(
"Added profile '" + profileName + "' with resources: " + resource);
}
}
// check to make sure mandatory profiles are present
@ -149,7 +159,7 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager {
+ Arrays.toString(MANDATORY_PROFILES) + " must be present");
}
}
LOG.info("Loaded profiles " + profiles.keySet());
LOG.info("Loaded profiles: " + profiles.keySet());
}
private Resource parseResource(Map profileInfo) throws IOException {
@ -182,13 +192,33 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager {
return resource;
}
@Override
public Resource getProfile(String profile) {
return Resources.clone(profiles.get(profile));
private void checkAndThrowExceptionWhenFeatureDisabled()
throws YARNFeatureNotEnabledException {
if (!profileEnabled) {
throw new YARNFeatureNotEnabledException(FEATURE_NOT_ENABLED_MSG);
}
}
@Override
public Map<String, Resource> getResourceProfiles() {
public Resource getProfile(String profile) throws YarnException{
checkAndThrowExceptionWhenFeatureDisabled();
if (profile == null) {
throw new YarnException("Profile name cannot be null");
}
Resource profileRes = profiles.get(profile);
if (profileRes == null) {
throw new YarnException(
"Resource profile '" + profile + "' not found");
}
return Resources.clone(profileRes);
}
@Override
public Map<String, Resource> getResourceProfiles()
throws YARNFeatureNotEnabledException {
checkAndThrowExceptionWhenFeatureDisabled();
return Collections.unmodifiableMap(profiles);
}
@ -200,17 +230,17 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager {
}
@Override
public Resource getDefaultProfile() {
public Resource getDefaultProfile() throws YarnException {
return getProfile(DEFAULT_PROFILE);
}
@Override
public Resource getMinimumProfile() {
public Resource getMinimumProfile() throws YarnException {
return getProfile(MINIMUM_PROFILE);
}
@Override
public Resource getMaximumProfile() {
public Resource getMaximumProfile() throws YarnException {
return getProfile(MAXIMUM_PROFILE);
}
@ -220,13 +250,4 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager {
Long.valueOf(value.substring(0, value.length() - units.length()));
return ResourceInformation.newInstance(name, units, resourceValue);
}
public List<ResourceTypeInfo> getAllResourceTypeInfo() {
try {
readLock.lock();
return Collections.unmodifiableList(resourceTypeInfo);
} finally {
readLock.unlock();
}
}
}

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
View File

@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEvent;
@ -1301,7 +1302,13 @@ public abstract class AbstractYarnScheduler
if (!profilesEnabled) {
ret = ResourceUtils.getResourceTypesMinimumAllocation();
} else {
ret = rmContext.getResourceProfilesManager().getMinimumProfile();
try {
ret = rmContext.getResourceProfilesManager().getMinimumProfile();
} catch (YarnException e) {
LOG.error(
"Exception while getting minimum profile from profile manager:", e);
throw new YarnRuntimeException(e);
}
}
LOG.info("Minimum allocation = " + ret);
return ret;
@ -1323,7 +1330,14 @@ public abstract class AbstractYarnScheduler
if (!profilesEnabled) {
ret = ResourceUtils.getResourceTypesMaximumAllocation();
} else {
ret = rmContext.getResourceProfilesManager().getMaximumProfile();
try {
ret = rmContext.getResourceProfilesManager().getMaximumProfile();
} catch (YarnException e) {
LOG.error(
"Exception while getting maximum profile from ResourceProfileManager:",
e);
throw new YarnRuntimeException(e);
}
}
LOG.info("Maximum allocation = " + ret);
return ret;

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
View File

@ -103,9 +103,11 @@ public class TestApplicationMasterService {
}
@Override
public void registerApplicationMaster(ApplicationAttemptId
applicationAttemptId, RegisterApplicationMasterRequest request,
RegisterApplicationMasterResponse response) throws IOException {
public void registerApplicationMaster(
ApplicationAttemptId applicationAttemptId,
RegisterApplicationMasterRequest request,
RegisterApplicationMasterResponse response)
throws IOException, YarnException {
nextProcessor.registerApplicationMaster(
applicationAttemptId, request, response);
}
@ -145,7 +147,8 @@ public class TestApplicationMasterService {
public void registerApplicationMaster(
ApplicationAttemptId applicationAttemptId,
RegisterApplicationMasterRequest request,
RegisterApplicationMasterResponse response) throws IOException {
RegisterApplicationMasterResponse response)
throws IOException, YarnException {
beforeRegCount.incrementAndGet();
nextProcessor.registerApplicationMaster(applicationAttemptId,
request, response);

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceProfiles.java
View File

@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.resource;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.junit.Assert;
import org.junit.Test;
@ -36,8 +37,13 @@ public class TestResourceProfiles {
Configuration conf = new Configuration();
// be default resource profiles should not be enabled
manager.init(conf);
Map<String, Resource> profiles = manager.getResourceProfiles();
Assert.assertTrue(profiles.isEmpty());
try {
manager.getResourceProfiles();
Assert
.fail("Exception should be thrown as resource profile is not enabled"
+ " and getResourceProfiles is invoked.");
} catch (YarnException ie) {
}
conf.setBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, true);
try {
manager.init(conf);