Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-7043. [Partial backport] Cleanup ResourceProfileManager. (wangda) 

Change-Id: I463356f37bf1f6a3f1fc3c594c79916e8c0ab913
(cherry picked from commit 39240b61a1)

Conflicts:
	hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
	hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
This commit is contained in:
Wangda Tan 2017-08-21 17:20:06 -07:00 committed by Jonathan Hung
parent 7c37fbef12
commit 1f05fa041a
13 changed files with 95 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceProcessor.java
View File

@ -52,11 +52,13 @@ public interface ApplicationMasterServiceProcessor {
* @param request Register Request.
* @param response Register Response.
* @throws IOException IOException.
* @throws YarnException in critical situation where invalid
* profiles/resources are added.
*/
void registerApplicationMaster(
ApplicationAttemptId applicationAttemptId,
void registerApplicationMaster(ApplicationAttemptId applicationAttemptId,
RegisterApplicationMasterRequest request,
RegisterApplicationMasterResponse response) throws IOException;
RegisterApplicationMasterResponse response)
throws IOException, YarnException;
/**
* Allocate call.

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java
View File

@ -77,6 +77,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YARNFeatureNotEnabledException;
/**
* <p>The protocol between clients and the <code>ResourceManager</code>
@ -598,8 +599,7 @@ public interface ApplicationClientProtocol extends ApplicationBaseProtocol {
* </p>
* @param request request to get the details of a resource profile
* @return Response containing the details for a particular resource profile
* @throws YarnException if resource profiles are not enabled on the RM or
* the profile cannot be found
* @throws YarnException if any error happens inside YARN
* @throws IOException in case of other errors
*/
@Public

45
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/YARNFeatureNotEnabledException.java Normal file
View File

@ -0,0 +1,45 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.exceptions;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
/**
* This exception is thrown when a feature is being used which is not enabled
* yet.
*/
@InterfaceAudience.Public
@InterfaceStability.Unstable
public class YARNFeatureNotEnabledException extends YarnException {
private static final long serialVersionUID = 898023752676L;
public YARNFeatureNotEnabledException(Throwable cause) {
super(cause);
}
public YARNFeatureNotEnabledException(String message) {
super(message);
}
public YARNFeatureNotEnabledException(String message, Throwable cause) {
super(message, cause);
}
}

21
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
View File

@ -28,6 +28,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
import org.apache.hadoop.yarn.conf.ConfigurationProvider;
import org.apache.hadoop.yarn.conf.ConfigurationProviderFactory;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@ -37,9 +38,12 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
@ -561,4 +565,21 @@ public class ResourceUtils {
}
return "";
}
/**
* Get all resource types information from known resource types.
* @return List of ResourceTypeInfo
*/
public static List<ResourceTypeInfo> getResourcesTypeInfo() {
List<ResourceTypeInfo> array = new ArrayList<>();
// Add all resource types
Collection<ResourceInformation> resourcesInfo =
ResourceUtils.getResourceTypes().values();
for (ResourceInformation resourceInfo : resourcesInfo) {
array.add(ResourceTypeInfo
.newInstance(resourceInfo.getName(), resourceInfo.getUnits(),
resourceInfo.getResourceType()));
}
return array;
}
}

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
View File

@ -72,6 +72,7 @@ import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
import org.apache.hadoop.yarn.exceptions.ApplicationIdNotProvidedException;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
import org.apache.hadoop.yarn.exceptions.YARNFeatureNotEnabledException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
@ -863,8 +864,7 @@ public abstract class YarnClient extends AbstractService {
* Get available resource types supported by RM.
* </p>
* @return list of supported resource types with detailed information
* @throws YarnException if resource profiles are not enabled or the profile
* cannot be found
* @throws YarnException if any issue happens inside YARN
* @throws IOException in case of other others
*/
@Public

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
View File

@ -528,10 +528,12 @@ public class DominantResourceCalculator extends ResourceCalculator {
long value;
if (stepFactorValue != 0) {
value = roundUp
? roundUp((long) Math.ceil(rValue * by), stepFactorValue)
? roundUp((long) Math.ceil((float) (rValue * by)), stepFactorValue)
: roundDown((long) (rValue * by), stepFactorValue);
} else {
value = roundUp ? (long) Math.ceil(rValue * by) : (long) (rValue * by);
value = roundUp
? (long) Math.ceil((float) (rValue * by))
: (long) (rValue * by);
}
tmp.setValue(value);
}

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
View File

@ -424,6 +424,7 @@ public class TestPBImplRecords extends BasePBImplRecordsTest {
generateByNewInstance(ContainerResourceIncreaseRequest.class);
generateByNewInstance(QueueConfigurations.class);
generateByNewInstance(CollectorInfo.class);
generateByNewInstance(ResourceTypeInfo.class);
}
@Test

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AMSProcessingChain.java
View File

@ -82,7 +82,7 @@ class AMSProcessingChain implements ApplicationMasterServiceProcessor {
public void registerApplicationMaster(
ApplicationAttemptId applicationAttemptId,
RegisterApplicationMasterRequest request,
RegisterApplicationMasterResponse resp) throws IOException {
RegisterApplicationMasterResponse resp) throws IOException, YarnException {
this.head.registerApplicationMaster(applicationAttemptId, request, resp);
}

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
View File

@ -176,6 +176,7 @@ import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.util.UTCClock;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import com.google.common.annotations.VisibleForTesting;
@ -1790,8 +1791,7 @@ public class ClientRMService extends AbstractService implements
GetAllResourceTypeInfoRequest request) throws YarnException, IOException {
GetAllResourceTypeInfoResponse response =
GetAllResourceTypeInfoResponse.newInstance();
response.setResourceTypeInfo(
resourceProfilesManager.getAllResourceTypeInfo());
response.setResourceTypeInfo(ResourceUtils.getResourcesTypeInfo());
return response;
}
}

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
View File

@ -111,7 +111,8 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor {
public void registerApplicationMaster(
ApplicationAttemptId applicationAttemptId,
RegisterApplicationMasterRequest request,
RegisterApplicationMasterResponse response) throws IOException {
RegisterApplicationMasterResponse response)
throws IOException, YarnException {
RMApp app = getRmContext().getRMApps().get(
applicationAttemptId.getApplicationId());

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
View File

@ -127,7 +127,8 @@ public class OpportunisticContainerAllocatorAMService
public void registerApplicationMaster(
ApplicationAttemptId applicationAttemptId,
RegisterApplicationMasterRequest request,
RegisterApplicationMasterResponse response) throws IOException {
RegisterApplicationMasterResponse response)
throws IOException, YarnException {
final SchedulerApplicationAttempt appAttempt = ((AbstractYarnScheduler)
getScheduler()).getApplicationAttempt(applicationAttemptId);
if (appAttempt.getOpportunisticContainerContext() == null) {

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
View File

@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEvent;

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
View File

@ -102,9 +102,11 @@ public class TestApplicationMasterService {
}
@Override
public void registerApplicationMaster(ApplicationAttemptId
applicationAttemptId, RegisterApplicationMasterRequest request,
RegisterApplicationMasterResponse response) throws IOException {
public void registerApplicationMaster(
ApplicationAttemptId applicationAttemptId,
RegisterApplicationMasterRequest request,
RegisterApplicationMasterResponse response)
throws IOException, YarnException {
nextProcessor.registerApplicationMaster(
applicationAttemptId, request, response);
}
@ -144,7 +146,8 @@ public class TestApplicationMasterService {
public void registerApplicationMaster(
ApplicationAttemptId applicationAttemptId,
RegisterApplicationMasterRequest request,
RegisterApplicationMasterResponse response) throws IOException {
RegisterApplicationMasterResponse response)
throws IOException, YarnException {
beforeRegCount.incrementAndGet();
nextProcessor.registerApplicationMaster(applicationAttemptId,
request, response);