From 09c12434a2f16a4f336a74474695bbe70097dc94 Mon Sep 17 00:00:00 2001 From: Bikas Saha Date: Thu, 6 Jun 2013 19:03:27 +0000 Subject: [PATCH] Merge r1490392 from trunk to branch-2 for YARN-750. Allow for black-listing resources in YARN API and Impl in CS (acmurthy via bikas) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1490396 13f79535-47bb-0310-9956-ffa450edef68 --- .../v2/app/local/LocalContainerAllocator.java | 2 +- .../v2/app/rm/RMContainerRequestor.java | 2 +- .../v2/app/TestRMContainerAllocator.java | 7 +- .../apache/hadoop/yarn/api/AMRMProtocol.java | 15 +- .../api/protocolrecords/AllocateRequest.java | 38 ++++- .../impl/pb/AllocateRequestPBImpl.java | 40 +++++ .../api/records/ResourceBlacklistRequest.java | 83 +++++++++ .../yarn/api/records/ResourceRequest.java | 8 +- .../pb/ResourceBlacklistRequestPBImpl.java | 159 ++++++++++++++++++ .../src/main/proto/yarn_protos.proto | 5 + .../src/main/proto/yarn_service_protos.proto | 5 +- .../hadoop/yarn/client/AMRMClientImpl.java | 2 +- .../ApplicationMasterService.java | 25 ++- .../rmapp/attempt/RMAppAttemptImpl.java | 4 +- .../scheduler/AppSchedulingInfo.java | 31 +++- ...alidResourceBlacklistRequestException.java | 47 ++++++ .../InvalidResourceRequestException.java | 9 +- .../scheduler/SchedulerUtils.java | 11 ++ .../scheduler/YarnScheduler.java | 6 +- .../scheduler/capacity/CapacityScheduler.java | 6 +- .../scheduler/capacity/LeafQueue.java | 27 +++ .../common/fica/FiCaSchedulerApp.java | 10 +- .../scheduler/fair/FSSchedulerApp.java | 2 +- .../scheduler/fair/FairScheduler.java | 2 +- .../scheduler/fifo/FifoScheduler.java | 4 +- .../server/resourcemanager/Application.java | 2 +- .../yarn/server/resourcemanager/MockAM.java | 2 +- .../resourcemanager/TestFifoScheduler.java | 8 +- .../TestAMRMRPCNodeUpdates.java | 14 +- .../TestAMRMRPCResponseId.java | 6 +- .../attempt/TestRMAppAttemptTransitions.java | 10 +- .../scheduler/TestSchedulerUtils.java | 102 +++++++++++ .../capacity/TestApplicationLimits.java | 6 +- .../scheduler/capacity/TestLeafQueue.java | 120 +++++++++---- .../scheduler/fair/TestFairScheduler.java | 18 +- .../scheduler/fifo/TestFifoScheduler.java | 2 +- .../server/TestContainerManagerSecurity.java | 2 +- 37 files changed, 745 insertions(+), 97 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceBlacklistRequest.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceBlacklistRequestPBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/InvalidResourceBlacklistRequestException.java diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java index 5a3f84fe452..d2fa70d9137 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java @@ -96,7 +96,7 @@ public class LocalContainerAllocator extends RMCommunicator AllocateRequest allocateRequest = AllocateRequest.newInstance( this.applicationAttemptId, this.lastResponseID, super .getApplicationProgress(), new ArrayList(), - new ArrayList()); + new ArrayList(), null); AllocateResponse allocateResponse; try { allocateResponse = scheduler.allocate(allocateRequest); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java index 59f113a4cf0..fff8415d2dd 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java @@ -148,7 +148,7 @@ public abstract class RMContainerRequestor extends RMCommunicator { AllocateRequest allocateRequest = AllocateRequest.newInstance( applicationAttemptId, lastResponseID, super.getApplicationProgress(), new ArrayList(ask), new ArrayList( - release)); + release), null); AllocateResponse allocateResponse; try { allocateResponse = scheduler.allocate(allocateRequest); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java index 63bd7b33a7f..c325940a039 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java @@ -1214,7 +1214,8 @@ public class TestRMContainerAllocator { @Override public synchronized Allocation allocate( ApplicationAttemptId applicationAttemptId, List ask, - List release) { + List release, + List blacklistAdditions, List blacklistRemovals) { List askCopy = new ArrayList(); for (ResourceRequest req : ask) { ResourceRequest reqCopy = ResourceRequest.newInstance(req @@ -1223,7 +1224,9 @@ public class TestRMContainerAllocator { askCopy.add(reqCopy); } lastAsk = ask; - return super.allocate(applicationAttemptId, askCopy, release); + return super.allocate( + applicationAttemptId, askCopy, release, + blacklistAdditions, blacklistRemovals); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/AMRMProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/AMRMProtocol.java index dedd9a9c3e4..b5f188df800 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/AMRMProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/AMRMProtocol.java @@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRespo import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.conf.YarnConfiguration; /** *

The protocol between a live instance of ApplicationMaster @@ -60,6 +61,8 @@ public interface AMRMProtocol { * @return registration respose * @throws YarnException * @throws IOException + * @see RegisterApplicationMasterRequest + * @see RegisterApplicationMasterResponse */ public RegisterApplicationMasterResponse registerApplicationMaster( RegisterApplicationMasterRequest request) @@ -80,6 +83,8 @@ public interface AMRMProtocol { * @return completion response * @throws YarnException * @throws IOException + * @see FinishApplicationMasterRequest + * @see FinishApplicationMasterResponse */ public FinishApplicationMasterResponse finishApplicationMaster( FinishApplicationMasterRequest request) @@ -91,12 +96,16 @@ public interface AMRMProtocol { * *

The ApplicationMaster uses this interface to provide a list * of {@link ResourceRequest} and returns unused {@link Container} allocated - * to it via {@link AllocateRequest}.

+ * to it via {@link AllocateRequest}. Optionally, the + * ApplicationMaster can also blacklist resources + * which it doesn't want to use.

* *

This also doubles up as a heartbeat to let the * ResourceManager know that the ApplicationMaster * is alive. Thus, applications should periodically make this call to be kept - * alive. The frequency depends on ??

+ * alive. The frequency depends on + * {@link YarnConfiguration#RM_AM_EXPIRY_INTERVAL_MS} which defaults to + * {@link YarnConfiguration#DEFAULT_RM_AM_EXPIRY_INTERVAL_MS}.

* *

The ResourceManager responds with list of allocated * {@link Container}, status of completed containers and headroom information @@ -110,6 +119,8 @@ public interface AMRMProtocol { * @return allocation response * @throws YarnException * @throws IOException + * @see AllocateRequest + * @see AllocateResponse */ public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java index 476792299bf..763e1f0a350 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java @@ -24,6 +24,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.yarn.api.AMRMProtocol; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ResourceRequest; @@ -61,13 +62,15 @@ public abstract class AllocateRequest { public static AllocateRequest newInstance( ApplicationAttemptId applicationAttemptId, int responseID, float appProgress, List resourceAsk, - List containersToBeReleased) { + List containersToBeReleased, + ResourceBlacklistRequest resourceBlacklistRequest) { AllocateRequest allocateRequest = Records.newRecord(AllocateRequest.class); allocateRequest.setApplicationAttemptId(applicationAttemptId); allocateRequest.setResponseId(responseID); allocateRequest.setProgress(appProgress); allocateRequest.setAskList(resourceAsk); allocateRequest.setReleaseList(containersToBeReleased); + allocateRequest.setResourceBlacklistRequest(resourceBlacklistRequest); return allocateRequest; } @@ -127,6 +130,7 @@ public abstract class AllocateRequest { * Get the list of ResourceRequest to update the * ResourceManager about the application's resource requirements. * @return the list of ResourceRequest + * @see ResourceRequest */ @Public @Stable @@ -138,6 +142,7 @@ public abstract class AllocateRequest { * @param resourceRequests list of ResourceRequest to update the * ResourceManager about the application's * resource requirements + * @see ResourceRequest */ @Public @Stable @@ -157,10 +162,37 @@ public abstract class AllocateRequest { * Set the list of ContainerId of containers being * released by the ApplicationMaster * @param releaseContainers list of ContainerId of - * containers being released by the < - * code>ApplicationMaster + * containers being released by the + * ApplicationMaster */ @Public @Stable public abstract void setReleaseList(List releaseContainers); + + /** + * Get the ResourceBlacklistRequest being sent by the + * ApplicationMaster. + * @return the ResourceBlacklistRequest being sent by the + * ApplicationMaster + * @see ResourceBlacklistRequest + */ + @Public + @Stable + public abstract ResourceBlacklistRequest getResourceBlacklistRequest(); + + /** + * Set the ResourceBlacklistRequest to inform the + * ResourceManager about the blacklist additions and removals + * per the ApplicationMaster. + * + * @param resourceBlacklistRequest the ResourceBlacklistRequest + * to inform the ResourceManager about + * the blacklist additions and removals + * per the ApplicationMaster + * @see ResourceBlacklistRequest + */ + @Public + @Stable + public abstract void setResourceBlacklistRequest( + ResourceBlacklistRequest resourceBlacklistRequest); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java index 2a78efa7bef..37f9f4ed626 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java @@ -25,12 +25,15 @@ import java.util.List; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.ResourceBlacklistRequestPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ResourceRequestPBImpl; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ResourceBlacklistRequestProto; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.AllocateRequestProto; @@ -46,6 +49,7 @@ public class AllocateRequestPBImpl extends AllocateRequest { private ApplicationAttemptId applicationAttemptID = null; private List ask = null; private List release = null; + private ResourceBlacklistRequest blacklistRequest = null; public AllocateRequestPBImpl() { @@ -94,6 +98,9 @@ public class AllocateRequestPBImpl extends AllocateRequest { if (this.release != null) { addReleasesToProto(); } + if (this.blacklistRequest != null) { + builder.setBlacklistRequest(convertToProtoFormat(this.blacklistRequest)); + } } private void mergeLocalToProto() { @@ -161,6 +168,7 @@ public class AllocateRequestPBImpl extends AllocateRequest { initAsks(); return this.ask; } + @Override public void setAskList(final List resourceRequests) { if(resourceRequests == null) { @@ -171,6 +179,28 @@ public class AllocateRequestPBImpl extends AllocateRequest { this.ask.addAll(resourceRequests); } + @Override + public ResourceBlacklistRequest getResourceBlacklistRequest() { + AllocateRequestProtoOrBuilder p = viaProto ? proto : builder; + if (this.blacklistRequest != null) { + return this.blacklistRequest; + } + if (!p.hasBlacklistRequest()) { + return null; + } + this.blacklistRequest = convertFromProtoFormat(p.getBlacklistRequest()); + return this.blacklistRequest; + } + + @Override + public void setResourceBlacklistRequest(ResourceBlacklistRequest blacklistRequest) { + maybeInitBuilder(); + if (blacklistRequest == null) { + builder.clearBlacklistRequest(); + } + this.blacklistRequest = blacklistRequest; + } + private void initAsks() { if (this.ask != null) { return; @@ -302,4 +332,14 @@ public class AllocateRequestPBImpl extends AllocateRequest { private ContainerIdProto convertToProtoFormat(ContainerId t) { return ((ContainerIdPBImpl)t).getProto(); } + + private ResourceBlacklistRequestPBImpl convertFromProtoFormat(ResourceBlacklistRequestProto p) { + return new ResourceBlacklistRequestPBImpl(p); + } + + private ResourceBlacklistRequestProto convertToProtoFormat(ResourceBlacklistRequest t) { + return ((ResourceBlacklistRequestPBImpl)t).getProto(); + } + + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceBlacklistRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceBlacklistRequest.java new file mode 100644 index 00000000000..60f12c36f05 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceBlacklistRequest.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.records; + +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.yarn.api.AMRMProtocol; +import org.apache.hadoop.yarn.util.Records; + +/** + * {@link ResourceBlacklistRequest} encapsulates the list of resource-names + * which should be added or removed from the blacklist of resources + * for the application. + * + * @see ResourceRequest + * @see AMRMProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest) + */ +@Public +@Stable +public abstract class ResourceBlacklistRequest { + + public static ResourceBlacklistRequest newInstance( + List additions, List removals) { + ResourceBlacklistRequest blacklistRequest = + Records.newRecord(ResourceBlacklistRequest.class); + blacklistRequest.setBlacklistAdditions(additions); + blacklistRequest.setBlacklistRemovals(removals); + return blacklistRequest; + } + + /** + * Get the list of resources which should be added to the + * application blacklist. + * + * @return list of resources which should be added to the + * application blacklist + */ + public abstract List getBlacklistAdditions(); + + /** + * Set list of resources which should be added to the application blacklist. + * + * @param resourceNames list of resources which should be added to the + * application blacklist + */ + public abstract void setBlacklistAdditions(List resourceNames); + + /** + * Get the list of resources which should be removed from the + * application blacklist. + * + * @return list of resources which should be removed from the + * application blacklist + */ + public abstract List getBlacklistRemovals(); + + /** + * Set list of resources which should be removed from the + * application blacklist. + * + * @param resourceNames list of resources which should be removed from the + * application blacklist + */ + public abstract void setBlacklistRemovals(List resourceNames); + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java index d4afda8df73..7f119fda549 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java @@ -40,9 +40,15 @@ import org.apache.hadoop.yarn.util.Records; * *

  • {@link Resource} required for each request.
    • *
  • - * Number of containers of such specifications which are required + * Number of containers, of above specifications, which are required * by the application. *
    • + *
  • + * A boolean relaxLocality flag, defaulting to true, + * which tells the ResourceManager if the application wants + * locality to be loose (i.e. allows fall-through to rack or any) + * or strict (i.e. specify hard constraint on resource allocation). + *
    • * *

    * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceBlacklistRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceBlacklistRequestPBImpl.java new file mode 100644 index 00000000000..583068b41a8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceBlacklistRequestPBImpl.java @@ -0,0 +1,159 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.impl.pb; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; +import org.apache.hadoop.yarn.proto.YarnProtos.ResourceBlacklistRequestProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ResourceBlacklistRequestProtoOrBuilder; + +public class ResourceBlacklistRequestPBImpl extends ResourceBlacklistRequest { + + ResourceBlacklistRequestProto proto = null; + ResourceBlacklistRequestProto.Builder builder = null; + boolean viaProto = false; + + List blacklistAdditions = null; + List blacklistRemovals = null; + + public ResourceBlacklistRequestPBImpl() { + builder = ResourceBlacklistRequestProto.newBuilder(); + } + + public ResourceBlacklistRequestPBImpl(ResourceBlacklistRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public ResourceBlacklistRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = ResourceBlacklistRequestProto.newBuilder(proto); + } + viaProto = false; + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void mergeLocalToBuilder() { + if (this.blacklistAdditions != null) { + addBlacklistAdditionsToProto(); + } + if (this.blacklistRemovals != null) { + addBlacklistRemovalsToProto(); + } + } + + private void addBlacklistAdditionsToProto() { + maybeInitBuilder(); + builder.clearBlacklistAdditions(); + if (this.blacklistAdditions == null) { + return; + } + builder.addAllBlacklistAdditions(this.blacklistAdditions); + } + + private void addBlacklistRemovalsToProto() { + maybeInitBuilder(); + builder.clearBlacklistAdditions(); + if (this.blacklistRemovals == null) { + return; + } + builder.addAllBlacklistRemovals(this.blacklistRemovals); + } + + private void initBlacklistAdditions() { + if (this.blacklistAdditions != null) { + return; + } + ResourceBlacklistRequestProtoOrBuilder p = viaProto ? proto : builder; + List list = p.getBlacklistAdditionsList(); + this.blacklistAdditions = new ArrayList(); + this.blacklistAdditions.addAll(list); + } + + private void initBlacklistRemovals() { + if (this.blacklistRemovals != null) { + return; + } + ResourceBlacklistRequestProtoOrBuilder p = viaProto ? proto : builder; + List list = p.getBlacklistRemovalsList(); + this.blacklistRemovals = new ArrayList(); + this.blacklistRemovals.addAll(list); + } + + @Override + public List getBlacklistAdditions() { + initBlacklistAdditions(); + return this.blacklistAdditions; + } + + @Override + public void setBlacklistAdditions(List resourceNames) { + if (resourceNames == null) { + if (this.blacklistAdditions != null) { + this.blacklistAdditions.clear(); + } + return; + } + initBlacklistAdditions(); + this.blacklistAdditions.clear(); + this.blacklistAdditions.addAll(resourceNames); + } + + @Override + public List getBlacklistRemovals() { + initBlacklistRemovals(); + return this.blacklistRemovals; + } + + @Override + public void setBlacklistRemovals(List resourceNames) { + if (resourceNames == null) { + if (this.blacklistRemovals != null) { + this.blacklistRemovals.clear(); + } + return; + } + initBlacklistRemovals(); + this.blacklistRemovals.clear(); + this.blacklistRemovals.addAll(resourceNames); + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index b5057cf9093..68575854c8b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -229,6 +229,11 @@ message PreemptionResourceRequestProto { optional ResourceRequestProto resource = 1; } +message ResourceBlacklistRequestProto { + repeated string blacklist_additions = 1; + repeated string blacklist_removals = 2; +} + //////////////////////////////////////////////////////////////////////// ////// From client_RM_Protocol ///////////////////////////////////////// //////////////////////////////////////////////////////////////////////// diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index d68f24ca6a2..a06b0ace9d8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -54,8 +54,9 @@ message AllocateRequestProto { optional ApplicationAttemptIdProto application_attempt_id = 1; repeated ResourceRequestProto ask = 2; repeated ContainerIdProto release = 3; - optional int32 response_id = 4; - optional float progress = 5; + optional ResourceBlacklistRequestProto blacklist_request = 4; + optional int32 response_id = 5; + optional float progress = 6; } message AllocateResponseProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClientImpl.java index eee0191b838..9562135cec2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClientImpl.java @@ -228,7 +228,7 @@ public class AMRMClientImpl release.clear(); allocateRequest = AllocateRequest.newInstance(appAttemptId, lastResponseId, - progressIndicator, askList, releaseList); + progressIndicator, askList, releaseList, null); } allocateResponse = rmClient.allocate(allocateRequest); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java index 538269901d5..b72a7f6df04 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java @@ -52,6 +52,7 @@ import org.apache.hadoop.yarn.api.records.PreemptionContract; import org.apache.hadoop.yarn.api.records.PreemptionMessage; import org.apache.hadoop.yarn.api.records.PreemptionResourceRequest; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.StrictPreemptionContract; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -69,6 +70,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAt import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUnregistrationEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.InvalidResourceBlacklistRequestException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.InvalidResourceRequestException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; @@ -286,18 +288,35 @@ public class ApplicationMasterService extends AbstractService implements List ask = request.getAskList(); List release = request.getReleaseList(); - + + ResourceBlacklistRequest blacklistRequest = request.getResourceBlacklistRequest(); + List blacklistAdditions = + (blacklistRequest != null) ? + blacklistRequest.getBlacklistAdditions() : null; + List blacklistRemovals = + (blacklistRequest != null) ? + blacklistRequest.getBlacklistRemovals() : null; + // sanity check try { SchedulerUtils.validateResourceRequests(ask, rScheduler.getMaximumResourceCapability()); } catch (InvalidResourceRequestException e) { LOG.warn("Invalid resource ask by application " + appAttemptId, e); - throw RPCUtil.getRemoteException(e); + throw e; } + + try { + SchedulerUtils.validateBlacklistRequest(blacklistRequest); + } catch (InvalidResourceBlacklistRequestException e) { + LOG.warn("Invalid blacklist request by application " + appAttemptId, e); + throw e; + } + // Send new requests to appAttempt. Allocation allocation = - this.rScheduler.allocate(appAttemptId, ask, release); + this.rScheduler.allocate(appAttemptId, ask, release, + blacklistAdditions, blacklistRemovals); RMApp app = this.rmContext.getRMApps().get( appAttemptId.getApplicationId()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 69f058a92ea..bb2e2529c5d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -803,7 +803,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { // AM resource has been checked when submission Allocation amContainerAllocation = appAttempt.scheduler.allocate( appAttempt.applicationAttemptId, - Collections.singletonList(request), EMPTY_CONTAINER_RELEASE_LIST); + Collections.singletonList(request), EMPTY_CONTAINER_RELEASE_LIST, null, null); if (amContainerAllocation != null && amContainerAllocation.getContainers() != null) { assert (amContainerAllocation.getContainers().size() == 0); @@ -827,7 +827,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { // Acquire the AM container from the scheduler. Allocation amContainerAllocation = appAttempt.scheduler.allocate( appAttempt.applicationAttemptId, EMPTY_CONTAINER_REQUEST_LIST, - EMPTY_CONTAINER_RELEASE_LIST); + EMPTY_CONTAINER_RELEASE_LIST, null, null); // Set the masterContainer appAttempt.setMasterContainer(amContainerAllocation.getContainers().get( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java index 6962a2c7316..416e1e6e4df 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java @@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -59,7 +60,8 @@ public class AppSchedulingInfo { new org.apache.hadoop.yarn.server.resourcemanager.resource.Priority.Comparator()); final Map> requests = new HashMap>(); - + final Set blacklist = new HashSet(); + //private final ApplicationStore store; private final ActiveUsersManager activeUsersManager; @@ -114,12 +116,15 @@ public class AppSchedulingInfo { * application, by asking for more resources and releasing resources acquired * by the application. * - * @param requests - * resources to be acquired + * @param requests resources to be acquired + * @param blacklistAdditions resources to be added to the blacklist + * @param blacklistRemovals resources to be removed from the blacklist */ synchronized public void updateResourceRequests( - List requests) { + List requests, + List blacklistAdditions, List blacklistRemovals) { QueueMetrics metrics = queue.getMetrics(); + // Update resource requests for (ResourceRequest request : requests) { Priority priority = request.getPriority(); @@ -175,6 +180,20 @@ public class AppSchedulingInfo { lastRequestContainers))); } } + + // + // Update blacklist + // + + // Add to blacklist + if (blacklistAdditions != null) { + blacklist.addAll(blacklistAdditions); + } + + // Remove from blacklist + if (blacklistRemovals != null) { + blacklist.removeAll(blacklistRemovals); + } } synchronized public Collection getPriorities() { @@ -197,6 +216,10 @@ public class AppSchedulingInfo { return request.getCapability(); } + public synchronized boolean isBlacklisted(String resourceName) { + return blacklist.contains(resourceName); + } + /** * Resources have been allocated to this application by the resource * scheduler. Track them. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/InvalidResourceBlacklistRequestException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/InvalidResourceBlacklistRequestException.java new file mode 100644 index 00000000000..cd698ad143e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/InvalidResourceBlacklistRequestException.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler; + +import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.exceptions.YarnException; + +/** + * The exception is thrown when an application provides an invalid + * specification for the blacklist. + * + * As an e.g., currently this exceptions is thrown when an application + * tries to blacklist {@link ResourceRequest#ANY}. + */ +public class InvalidResourceBlacklistRequestException extends YarnException { + + private static final long serialVersionUID = 384957911L; + + public InvalidResourceBlacklistRequestException(Throwable cause) { + super(cause); + } + + public InvalidResourceBlacklistRequestException(String message) { + super(message); + } + + public InvalidResourceBlacklistRequestException(String message, Throwable cause) { + super(message, cause); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/InvalidResourceRequestException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/InvalidResourceRequestException.java index 27628f3703f..0e1a2c693b9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/InvalidResourceRequestException.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/InvalidResourceRequestException.java @@ -18,14 +18,17 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; -import org.apache.hadoop.yarn.YarnRuntimeException; +import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.exceptions.YarnException; /** - * The exception is thrown when the requested resource is out of the range + * The exception is thrown when a {@link ResourceRequest} is out of the range * of the configured lower and upper resource boundaries. * */ -public class InvalidResourceRequestException extends YarnRuntimeException { +public class InvalidResourceRequestException extends YarnException { + + private static final long serialVersionUID = 13498237L; public InvalidResourceRequestException(Throwable cause) { super(cause); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java index c0a54c79c70..e4e15b3f945 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java @@ -22,6 +22,7 @@ import java.util.List; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.ContainerExitStatus; +import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; @@ -152,4 +153,14 @@ public class SchedulerUtils { } } + public static void validateBlacklistRequest(ResourceBlacklistRequest blacklistRequest) + throws InvalidResourceBlacklistRequestException { + if (blacklistRequest != null) { + List plus = blacklistRequest.getBlacklistAdditions(); + if (plus != null && plus.contains(ResourceRequest.ANY)) { + throw new InvalidResourceBlacklistRequestException( + "Cannot add " + ResourceRequest.ANY + " to the blacklist!"); + } + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java index f0846497235..08f667c2f26 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java @@ -95,6 +95,8 @@ public interface YarnScheduler extends EventHandler { * @param appAttemptId * @param ask * @param release + * @param blacklistAdditions + * @param blacklistRemovals * @return the {@link Allocation} for the application */ @Public @@ -102,7 +104,9 @@ public interface YarnScheduler extends EventHandler { Allocation allocate(ApplicationAttemptId appAttemptId, List ask, - List release); + List release, + List blacklistAdditions, + List blacklistRemovals); /** * Get node resource usage report. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index aca2a1292e9..4ebf94b328f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -472,7 +472,8 @@ implements ResourceScheduler, CapacitySchedulerContext, Configurable { @Override @Lock(Lock.NoLock.class) public Allocation allocate(ApplicationAttemptId applicationAttemptId, - List ask, List release) { + List ask, List release, + List blacklistAdditions, List blacklistRemovals) { FiCaSchedulerApp application = getApplication(applicationAttemptId); if (application == null) { @@ -523,7 +524,8 @@ implements ResourceScheduler, CapacitySchedulerContext, Configurable { application.showRequests(); // Update application requests - application.updateResourceRequests(ask); + application.updateResourceRequests(ask, + blacklistAdditions, blacklistRemovals); LOG.debug("allocate: post-update"); application.showRequests(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 129d15c8f25..130d8b6684e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -815,6 +815,11 @@ public class LeafQueue implements CSQueue { } synchronized (application) { + // Check if this resource is on the blacklist + if (isBlacklisted(application, node)) { + continue; + } + // Schedule in priority order for (Priority priority : application.getPriorities()) { // Required resource @@ -897,6 +902,28 @@ public class LeafQueue implements CSQueue { return NULL_ASSIGNMENT; } + + boolean isBlacklisted(FiCaSchedulerApp application, FiCaSchedulerNode node) { + if (application.isBlacklisted(node.getHostName())) { + if (LOG.isDebugEnabled()) { + LOG.debug("Skipping 'host' " + node.getHostName() + + " for " + application.getApplicationId() + + " since it has been blacklisted"); + } + return true; + } + + if (application.isBlacklisted(node.getRackName())) { + if (LOG.isDebugEnabled()) { + LOG.debug("Skipping 'rack' " + node.getRackName() + + " for " + application.getApplicationId() + + " since it has been blacklisted"); + } + return true; + } + + return false; + } private synchronized CSAssignment assignReservedContainer(FiCaSchedulerApp application, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java index ba4efaa7c62..86bed0d8a8b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java @@ -134,9 +134,11 @@ public class FiCaSchedulerApp extends SchedulerApplication { } public synchronized void updateResourceRequests( - List requests) { + List requests, + List blacklistAdditions, List blacklistRemovals) { if (!isStopped) { - this.appSchedulingInfo.updateResourceRequests(requests); + this.appSchedulingInfo.updateResourceRequests(requests, + blacklistAdditions, blacklistRemovals); } } @@ -163,6 +165,10 @@ public class FiCaSchedulerApp extends SchedulerApplication { public Resource getResource(Priority priority) { return this.appSchedulingInfo.getResource(priority); } + + public boolean isBlacklisted(String resourceName) { + return this.appSchedulingInfo.isBlacklisted(resourceName); + } /** * Is this application pending? diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java index 9db15a6bc0b..4aaf74b87b7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java @@ -138,7 +138,7 @@ public class FSSchedulerApp extends SchedulerApplication { public synchronized void updateResourceRequests( List requests) { - this.appSchedulingInfo.updateResourceRequests(requests); + this.appSchedulingInfo.updateResourceRequests(requests, null, null); } public Map getResourceRequests(Priority priority) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index f8cff0ef258..b996952cb4c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -718,7 +718,7 @@ public class FairScheduler implements ResourceScheduler { @Override public Allocation allocate(ApplicationAttemptId appAttemptId, - List ask, List release) { + List ask, List release, List blacklistAdditions, List blacklistRemovals) { // Make sure this application exists FSSchedulerApp application = applications.get(appAttemptId); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index 0de8c59f82a..e7a2e94eafa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -222,7 +222,7 @@ public class FifoScheduler implements ResourceScheduler, Configurable { @Override public Allocation allocate( ApplicationAttemptId applicationAttemptId, List ask, - List release) { + List release, List blacklistAdditions, List blacklistRemovals) { FiCaSchedulerApp application = getApplication(applicationAttemptId); if (application == null) { LOG.error("Calling allocate on removed " + @@ -268,7 +268,7 @@ public class FifoScheduler implements ResourceScheduler, Configurable { application.showRequests(); // Update application requests - application.updateResourceRequests(ask); + application.updateResourceRequests(ask, null, null); LOG.debug("allocate: post-update" + " applicationId=" + applicationAttemptId + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java index af626aad9ed..56fc119c1f7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java @@ -265,7 +265,7 @@ public class Application { // Get resources from the ResourceManager resourceManager.getResourceScheduler().allocate(applicationAttemptId, - new ArrayList(ask), new ArrayList()); + new ArrayList(ask), new ArrayList(), null, null); System.out.println("-=======" + applicationAttemptId); System.out.println("----------" + resourceManager.getRMContext().getRMApps() .get(applicationId).getRMAppAttempt(applicationAttemptId)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java index 9e1072e40e1..97f392e8990 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java @@ -144,7 +144,7 @@ public class MockAM { List resourceRequest, List releases) throws Exception { AllocateRequest req = AllocateRequest.newInstance(attemptId, - ++responseId, 0F, resourceRequest, releases); + ++responseId, 0F, resourceRequest, releases, null); return amRMProtocol.allocate(req); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java index aa5e28671b7..160d8d40864 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java @@ -273,23 +273,23 @@ public class TestFifoScheduler { List ask1 = new ArrayList(); ask1.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), ResourceRequest.ANY, BuilderUtils.newResource(GB, 1), 1)); - fs.allocate(appAttemptId1, ask1, emptyId); + fs.allocate(appAttemptId1, ask1, emptyId, null, null); // Ask for a 2 GB container for app 2 List ask2 = new ArrayList(); ask2.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), ResourceRequest.ANY, BuilderUtils.newResource(2 * GB, 1), 1)); - fs.allocate(appAttemptId2, ask2, emptyId); + fs.allocate(appAttemptId2, ask2, emptyId, null, null); // Trigger container assignment fs.handle(new NodeUpdateSchedulerEvent(n1)); // Get the allocation for the applications and verify headroom - Allocation allocation1 = fs.allocate(appAttemptId1, emptyAsk, emptyId); + Allocation allocation1 = fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null); Assert.assertEquals("Allocation headroom", 1 * GB, allocation1.getResourceLimit().getMemory()); - Allocation allocation2 = fs.allocate(appAttemptId2, emptyAsk, emptyId); + Allocation allocation2 = fs.allocate(appAttemptId2, emptyAsk, emptyId, null, null); Assert.assertEquals("Allocation headroom", 1 * GB, allocation2.getResourceLimit().getMemory()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java index 4f21cb16ad3..eb234b268b2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java @@ -108,7 +108,7 @@ public class TestAMRMRPCNodeUpdates { // allocate request returns no updated node AllocateRequest allocateRequest1 = AllocateRequest.newInstance(attempt1 - .getAppAttemptId(), 0, 0F, null, null); + .getAppAttemptId(), 0, 0F, null, null, null); AllocateResponse response1 = amService.allocate(allocateRequest1); List updatedNodes = response1.getUpdatedNodes(); Assert.assertEquals(0, updatedNodes.size()); @@ -117,7 +117,7 @@ public class TestAMRMRPCNodeUpdates { // allocate request returns updated node allocateRequest1 = AllocateRequest.newInstance(attempt1 - .getAppAttemptId(), response1.getResponseId(), 0F, null, null); + .getAppAttemptId(), response1.getResponseId(), 0F, null, null, null); response1 = amService.allocate(allocateRequest1); updatedNodes = response1.getUpdatedNodes(); Assert.assertEquals(1, updatedNodes.size()); @@ -137,7 +137,7 @@ public class TestAMRMRPCNodeUpdates { // subsequent allocate request returns delta allocateRequest1 = AllocateRequest.newInstance(attempt1 - .getAppAttemptId(), response1.getResponseId(), 0F, null, null); + .getAppAttemptId(), response1.getResponseId(), 0F, null, null, null); response1 = amService.allocate(allocateRequest1); updatedNodes = response1.getUpdatedNodes(); Assert.assertEquals(1, updatedNodes.size()); @@ -157,7 +157,7 @@ public class TestAMRMRPCNodeUpdates { // allocate request returns no updated node AllocateRequest allocateRequest2 = AllocateRequest.newInstance(attempt2 - .getAppAttemptId(), 0, 0F, null, null); + .getAppAttemptId(), 0, 0F, null, null, null); AllocateResponse response2 = amService.allocate(allocateRequest2); updatedNodes = response2.getUpdatedNodes(); Assert.assertEquals(0, updatedNodes.size()); @@ -166,7 +166,7 @@ public class TestAMRMRPCNodeUpdates { // both AM's should get delta updated nodes allocateRequest1 = AllocateRequest.newInstance(attempt1 - .getAppAttemptId(), response1.getResponseId(), 0F, null, null); + .getAppAttemptId(), response1.getResponseId(), 0F, null, null, null); response1 = amService.allocate(allocateRequest1); updatedNodes = response1.getUpdatedNodes(); Assert.assertEquals(1, updatedNodes.size()); @@ -175,7 +175,7 @@ public class TestAMRMRPCNodeUpdates { Assert.assertEquals(NodeState.RUNNING, nr.getNodeState()); allocateRequest2 = AllocateRequest.newInstance(attempt2 - .getAppAttemptId(), response2.getResponseId(), 0F, null, null); + .getAppAttemptId(), response2.getResponseId(), 0F, null, null, null); response2 = amService.allocate(allocateRequest2); updatedNodes = response2.getUpdatedNodes(); Assert.assertEquals(1, updatedNodes.size()); @@ -185,7 +185,7 @@ public class TestAMRMRPCNodeUpdates { // subsequent allocate calls should return no updated nodes allocateRequest2 = AllocateRequest.newInstance(attempt2 - .getAppAttemptId(), response2.getResponseId(), 0F, null, null); + .getAppAttemptId(), response2.getResponseId(), 0F, null, null, null); response2 = amService.allocate(allocateRequest2); updatedNodes = response2.getUpdatedNodes(); Assert.assertEquals(0, updatedNodes.size()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java index d204a45b228..6e4019d832d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java @@ -78,13 +78,13 @@ public class TestAMRMRPCResponseId { am.registerAppAttempt(); AllocateRequest allocateRequest = AllocateRequest.newInstance(attempt - .getAppAttemptId(), 0, 0F, null, null); + .getAppAttemptId(), 0, 0F, null, null, null); AllocateResponse response = amService.allocate(allocateRequest); Assert.assertEquals(1, response.getResponseId()); Assert.assertFalse(response.getResync()); allocateRequest = AllocateRequest.newInstance(attempt - .getAppAttemptId(), response.getResponseId(), 0F, null, null); + .getAppAttemptId(), response.getResponseId(), 0F, null, null, null); response = amService.allocate(allocateRequest); Assert.assertEquals(2, response.getResponseId()); @@ -94,7 +94,7 @@ public class TestAMRMRPCResponseId { /** try sending old request again **/ allocateRequest = AllocateRequest.newInstance(attempt - .getAppAttemptId(), 0, 0F, null, null); + .getAppAttemptId(), 0, 0F, null, null, null); response = amService.allocate(allocateRequest); Assert.assertTrue(response.getResync()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index f3ce664b477..4235e9d8460 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -330,7 +330,7 @@ public class TestRMAppAttemptTransitions { applicationAttempt.getAppAttemptState()); verify(scheduler, times(expectedAllocateCount)). allocate(any(ApplicationAttemptId.class), - any(List.class), any(List.class)); + any(List.class), any(List.class), any(List.class), any(List.class)); assertEquals(0,applicationAttempt.getJustFinishedContainers().size()); assertNull(applicationAttempt.getMasterContainer()); @@ -345,6 +345,7 @@ public class TestRMAppAttemptTransitions { /** * {@link RMAppAttemptState#ALLOCATED} */ + @SuppressWarnings("unchecked") private void testAppAttemptAllocatedState(Container amContainer) { assertEquals(RMAppAttemptState.ALLOCATED, applicationAttempt.getAppAttemptState()); @@ -354,7 +355,9 @@ public class TestRMAppAttemptTransitions { verify(applicationMasterLauncher).handle(any(AMLauncherEvent.class)); verify(scheduler, times(2)). allocate( - any(ApplicationAttemptId.class), any(List.class), any(List.class)); + any( + ApplicationAttemptId.class), any(List.class), any(List.class), + any(List.class), any(List.class)); } /** @@ -465,6 +468,7 @@ public class TestRMAppAttemptTransitions { testAppAttemptScheduledState(); } + @SuppressWarnings("unchecked") private Container allocateApplicationAttempt() { scheduleApplicationAttempt(); @@ -481,6 +485,8 @@ public class TestRMAppAttemptTransitions { scheduler.allocate( any(ApplicationAttemptId.class), any(List.class), + any(List.class), + any(List.class), any(List.class))). thenReturn(allocation); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java index f6484dd6c2c..22279484076 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java @@ -22,20 +22,47 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; +import java.security.PrivilegedAction; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.AMRMProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; +import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest; +import org.apache.hadoop.yarn.api.records.ApplicationAccessType; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.impl.pb.ResourceRequestPBImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.server.resourcemanager.MockNM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MockRMWithAMS; +import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MyContainerManager; import org.apache.hadoop.yarn.server.resourcemanager.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.server.resourcemanager.resource.DominantResourceCalculator; import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceCalculator; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.utils.BuilderUtils; +import org.apache.hadoop.yarn.util.Records; +import org.junit.Assert; import org.junit.Test; public class TestSchedulerUtils { + private static final Log LOG = LogFactory.getLog(TestSchedulerUtils.class); + @Test (timeout = 30000) public void testNormalizeRequest() { ResourceCalculator resourceCalculator = new DefaultResourceCalculator(); @@ -239,5 +266,80 @@ public class TestSchedulerUtils { // expected } } + + @Test + public void testValidateResourceBlacklistRequest() throws Exception { + + MyContainerManager containerManager = new MyContainerManager(); + final MockRM rm = + new MockRMWithAMS(new YarnConfiguration(), containerManager); + rm.start(); + + MockNM nm1 = rm.registerNode("localhost:1234", 5120); + + Map acls = + new HashMap(2); + acls.put(ApplicationAccessType.VIEW_APP, "*"); + RMApp app = rm.submitApp(1024, "appname", "appuser", acls); + + nm1.nodeHeartbeat(true); + + RMAppAttempt attempt = app.getCurrentAppAttempt(); + ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId(); + waitForLaunchedState(attempt); + + // Create a client to the RM. + final Configuration conf = rm.getConfig(); + final YarnRPC rpc = YarnRPC.create(conf); + + UserGroupInformation currentUser = + UserGroupInformation.createRemoteUser(applicationAttemptId.toString()); + + AMRMProtocol client = currentUser + .doAs(new PrivilegedAction() { + @Override + public AMRMProtocol run() { + return (AMRMProtocol) rpc.getProxy(AMRMProtocol.class, rm + .getApplicationMasterService().getBindAddress(), conf); + } + }); + + RegisterApplicationMasterRequest request = Records + .newRecord(RegisterApplicationMasterRequest.class); + request.setApplicationAttemptId(applicationAttemptId); + client.registerApplicationMaster(request); + + ResourceBlacklistRequest blacklistRequest = + ResourceBlacklistRequest.newInstance( + Collections.singletonList(ResourceRequest.ANY), null); + + AllocateRequest allocateRequest = + AllocateRequest.newInstance(applicationAttemptId, 0, 0.0f, null, null, + blacklistRequest); + boolean error = false; + try { + client.allocate(allocateRequest); + } catch (InvalidResourceBlacklistRequestException e) { + error = true; + } + + rm.stop(); + + Assert.assertTrue( + "Didn't not catch InvalidResourceBlacklistRequestException", error); + } + + private void waitForLaunchedState(RMAppAttempt attempt) + throws InterruptedException { + int waitCount = 0; + while (attempt.getAppAttemptState() != RMAppAttemptState.LAUNCHED + && waitCount++ < 20) { + LOG.info("Waiting for AppAttempt to reach LAUNCHED state. " + + "Current state is " + attempt.getAppAttemptState()); + Thread.sleep(1000); + } + Assert.assertEquals(attempt.getAppAttemptState(), + RMAppAttemptState.LAUNCHED); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java index 84d9236b406..f8dd885261e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java @@ -513,7 +513,7 @@ public class TestApplicationLimits { app_0_0_requests.add( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true, priority_1, recordFactory)); - app_0_0.updateResourceRequests(app_0_0_requests); + app_0_0.updateResourceRequests(app_0_0_requests, null, null); // Schedule to compute queue.assignContainers(clusterResource, node_0); @@ -532,7 +532,7 @@ public class TestApplicationLimits { app_0_1_requests.add( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true, priority_1, recordFactory)); - app_0_1.updateResourceRequests(app_0_1_requests); + app_0_1.updateResourceRequests(app_0_1_requests, null, null); // Schedule to compute queue.assignContainers(clusterResource, node_0); // Schedule to compute @@ -551,7 +551,7 @@ public class TestApplicationLimits { app_1_0_requests.add( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true, priority_1, recordFactory)); - app_1_0.updateResourceRequests(app_1_0_requests); + app_1_0.updateResourceRequests(app_1_0_requests, null, null); // Schedule to compute queue.assignContainers(clusterResource, node_0); // Schedule to compute diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 908e9a8ce32..d5f089e0ba5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -43,6 +43,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; @@ -293,7 +294,7 @@ public class TestLeafQueue { Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 3, true, - priority, recordFactory))); + priority, recordFactory)), null, null); // Start testing... @@ -415,11 +416,11 @@ public class TestLeafQueue { Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 3, true, - priority, recordFactory))); + priority, recordFactory)), null, null); app_1.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true, - priority, recordFactory))); + priority, recordFactory)), null, null); // Start testing... @@ -548,11 +549,11 @@ public class TestLeafQueue { Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, true, - priority, recordFactory))); + priority, recordFactory)), null, null); app_1.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true, - priority, recordFactory))); + priority, recordFactory)), null, null); /** * Start testing... @@ -641,11 +642,11 @@ public class TestLeafQueue { Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, true, - priority, recordFactory))); + priority, recordFactory)), null, null); app_1.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true, - priority, recordFactory))); + priority, recordFactory)), null, null); /** * Start testing... @@ -680,7 +681,7 @@ public class TestLeafQueue { a.setMaxCapacity(.1f); app_2.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, true, - priority, recordFactory))); + priority, recordFactory)), null, null); assertEquals(2, a.getActiveUsersManager().getNumActiveUsers()); // No more to user_0 since he is already over user-limit @@ -697,7 +698,7 @@ public class TestLeafQueue { LOG.info("here"); app_1.updateResourceRequests(Collections.singletonList( // unset TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 0, true, - priority, recordFactory))); + priority, recordFactory)), null, null); assertEquals(1, a.getActiveUsersManager().getNumActiveUsers()); a.assignContainers(clusterResource, node_1); assertEquals(1*GB, app_2.getHeadroom().getMemory()); // hit queue max-cap @@ -758,11 +759,11 @@ public class TestLeafQueue { Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 10, true, - priority, recordFactory))); + priority, recordFactory)), null, null); app_1.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 10, true, - priority, recordFactory))); + priority, recordFactory)), null, null); /** * Start testing... @@ -792,11 +793,11 @@ public class TestLeafQueue { app_2.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 3*GB, 1, true, - priority, recordFactory))); + priority, recordFactory)), null, null); app_3.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true, - priority, recordFactory))); + priority, recordFactory)), null, null); // Now allocations should goto app_2 since // user_0 is at limit inspite of high user-limit-factor @@ -920,11 +921,11 @@ public class TestLeafQueue { Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true, - priority, recordFactory))); + priority, recordFactory)), null, null); app_1.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 4*GB, 1, true, - priority, recordFactory))); + priority, recordFactory)), null, null); // Start testing... @@ -1022,7 +1023,7 @@ public class TestLeafQueue { Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, true, - priority, recordFactory))); + priority, recordFactory)), null, null); // Setup app_1 to request a 4GB container on host_0 and // another 4GB container anywhere. @@ -1034,7 +1035,7 @@ public class TestLeafQueue { true, priority, recordFactory)); appRequests_1.add(TestUtils.createResourceRequest(ResourceRequest.ANY, 4*GB, 2, true, priority, recordFactory)); - app_1.updateResourceRequests(appRequests_1); + app_1.updateResourceRequests(appRequests_1, null, null); // Start testing... @@ -1128,11 +1129,11 @@ public class TestLeafQueue { Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true, - priority, recordFactory))); + priority, recordFactory)), null, null); app_1.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 4*GB, 1, true, - priority, recordFactory))); + priority, recordFactory)), null, null); // Start testing... @@ -1255,7 +1256,7 @@ public class TestLeafQueue { app_0_requests_0.add( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 3, // one extra true, priority, recordFactory)); - app_0.updateResourceRequests(app_0_requests_0); + app_0.updateResourceRequests(app_0_requests_0, null, null); // Start testing... CSAssignment assignment = null; @@ -1320,7 +1321,7 @@ public class TestLeafQueue { app_0_requests_0.add( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, // one extra true, priority, recordFactory)); - app_0.updateResourceRequests(app_0_requests_0); + app_0.updateResourceRequests(app_0_requests_0, null, null); assertEquals(2, app_0.getTotalRequiredResources(priority)); String host_3 = "127.0.0.4"; // on rack_1 @@ -1411,7 +1412,7 @@ public class TestLeafQueue { TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, true, priority_2, recordFactory)); - app_0.updateResourceRequests(app_0_requests_0); + app_0.updateResourceRequests(app_0_requests_0, null, null); // Start testing... @@ -1526,7 +1527,7 @@ public class TestLeafQueue { app_0_requests_0.add( TestUtils.createResourceRequest(rack_1, 1*GB, 1, true, priority, recordFactory)); - app_0.updateResourceRequests(app_0_requests_0); + app_0.updateResourceRequests(app_0_requests_0, null, null); // Start testing... @@ -1535,7 +1536,7 @@ public class TestLeafQueue { app_0_requests_0.add( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, // only one true, priority, recordFactory)); - app_0.updateResourceRequests(app_0_requests_0); + app_0.updateResourceRequests(app_0_requests_0, null, null); // NODE_LOCAL - node_0_1 a.assignContainers(clusterResource, node_0_0); @@ -1558,7 +1559,7 @@ public class TestLeafQueue { app_0_requests_0.add( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, // only one true, priority, recordFactory)); - app_0.updateResourceRequests(app_0_requests_0); + app_0.updateResourceRequests(app_0_requests_0, null, null); // No allocation on node_0_1 even though it's node/rack local since // required(rack_1) == 0 @@ -1759,7 +1760,6 @@ public class TestLeafQueue { // Setup some nodes and racks String host_0_0 = "127.0.0.1"; String rack_0 = "rack_0"; - FiCaSchedulerNode node_0_0 = TestUtils.getMockNode(host_0_0, rack_0, 0, 8*GB); String host_0_1 = "127.0.0.2"; FiCaSchedulerNode node_0_1 = TestUtils.getMockNode(host_0_1, rack_0, 0, 8*GB); @@ -1789,6 +1789,7 @@ public class TestLeafQueue { // host_0_1: 8G // host_1_0: 8G // host_1_1: 8G + // Blacklist: Priority priority = TestUtils.createMockPriority(1); List app_0_requests_0 = new ArrayList(); app_0_requests_0.add( @@ -1803,7 +1804,8 @@ public class TestLeafQueue { app_0_requests_0.add( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, // only one false, priority, recordFactory)); - app_0.updateResourceRequests(app_0_requests_0); + app_0.updateResourceRequests(app_0_requests_0, + Collections.singletonList(host_0_0), null); app_0_requests_0.clear(); // @@ -1830,6 +1832,7 @@ public class TestLeafQueue { // host_0_1: 8G // host_1_0: 8G // host_1_1: 8G + // Blacklist: // node_1_1 // Shouldn't allocate since RR(rack_1) = relax: false @@ -1838,11 +1841,12 @@ public class TestLeafQueue { any(Priority.class), any(ResourceRequest.class), any(Container.class)); assertEquals(0, app_0.getSchedulingOpportunities(priority)); // should be 0 - // Allow rack-locality for rack_1 + // Allow rack-locality for rack_1, but blacklist node_1_1 app_0_requests_0.add( TestUtils.createResourceRequest(rack_1, 1*GB, 1, true, priority, recordFactory)); - app_0.updateResourceRequests(app_0_requests_0); + app_0.updateResourceRequests(app_0_requests_0, + Collections.singletonList(host_1_1), null); app_0_requests_0.clear(); // resourceName: @@ -1851,15 +1855,69 @@ public class TestLeafQueue { // rack_0: < null > // host_1_0: < 1, 1GB, 1, true > // host_1_1: < null > - // rack_1: < 1, 1GB, 1, true > <---- + // rack_1: < 1, 1GB, 1, true > // ANY: < 1, 1GB, 1, false > // Availability: // host_0_0: 8G // host_0_1: 8G // host_1_0: 8G // host_1_1: 8G + // Blacklist: < host_0_0 , host_1_1 > <---- // node_1_1 + // Shouldn't allocate since node_1_1 is blacklisted + a.assignContainers(clusterResource, node_1_1); + verify(app_0, never()).allocate(any(NodeType.class), eq(node_1_1), + any(Priority.class), any(ResourceRequest.class), any(Container.class)); + assertEquals(0, app_0.getSchedulingOpportunities(priority)); // should be 0 + + // Now, remove node_1_1 from blacklist, but add rack_1 to blacklist + app_0.updateResourceRequests(app_0_requests_0, + Collections.singletonList(rack_1), Collections.singletonList(host_1_1)); + app_0_requests_0.clear(); + + // resourceName: + // host_0_0: < 1, 1GB, 1, true > + // host_0_1: < null > + // rack_0: < null > + // host_1_0: < 1, 1GB, 1, true > + // host_1_1: < null > + // rack_1: < 1, 1GB, 1, true > + // ANY: < 1, 1GB, 1, false > + // Availability: + // host_0_0: 8G + // host_0_1: 8G + // host_1_0: 8G + // host_1_1: 8G + // Blacklist: < host_0_0 , rack_1 > <---- + + // node_1_1 + // Shouldn't allocate since rack_1 is blacklisted + a.assignContainers(clusterResource, node_1_1); + verify(app_0, never()).allocate(any(NodeType.class), eq(node_1_1), + any(Priority.class), any(ResourceRequest.class), any(Container.class)); + assertEquals(0, app_0.getSchedulingOpportunities(priority)); // should be 0 + + // Now remove rack_1 from blacklist + app_0.updateResourceRequests(app_0_requests_0, + null, Collections.singletonList(rack_1)); + app_0_requests_0.clear(); + + // resourceName: + // host_0_0: < 1, 1GB, 1, true > + // host_0_1: < null > + // rack_0: < null > + // host_1_0: < 1, 1GB, 1, true > + // host_1_1: < null > + // rack_1: < 1, 1GB, 1, true > + // ANY: < 1, 1GB, 1, false > + // Availability: + // host_0_0: 8G + // host_0_1: 8G + // host_1_0: 8G + // host_1_1: 8G + // Blacklist: < host_0_0 > <---- + // Now, should allocate since RR(rack_1) = relax: true a.assignContainers(clusterResource, node_1_1); verify(app_0).allocate(eq(NodeType.RACK_LOCAL), eq(node_1_1), @@ -1874,7 +1932,7 @@ public class TestLeafQueue { app_0_requests_0.add( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, // only one false, priority, recordFactory)); - app_0.updateResourceRequests(app_0_requests_0); + app_0.updateResourceRequests(app_0_requests_0, null, null); app_0_requests_0.clear(); // resourceName: diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 633d64db150..a8b764a6a9e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -206,7 +206,7 @@ public class TestFairScheduler { ResourceRequest request = createResourceRequest(memory, vcores, ResourceRequest.ANY, priority, numContainers, true); ask.add(request); - scheduler.allocate(id, ask, new ArrayList()); + scheduler.allocate(id, ask, new ArrayList(), null, null); return id; } @@ -221,7 +221,7 @@ public class TestFairScheduler { ApplicationAttemptId attId) { List ask = new ArrayList(); ask.add(request); - scheduler.allocate(attId, ask, new ArrayList()); + scheduler.allocate(attId, ask, new ArrayList(), null, null); } // TESTS @@ -528,7 +528,7 @@ public class TestFairScheduler { ResourceRequest request1 = createResourceRequest(minReqSize * 2, ResourceRequest.ANY, 1, 1, true); ask1.add(request1); - scheduler.allocate(id11, ask1, new ArrayList()); + scheduler.allocate(id11, ask1, new ArrayList(), null, null); // Second ask, queue2 requests 1 large + (2 * minReqSize) List ask2 = new ArrayList(); @@ -538,14 +538,14 @@ public class TestFairScheduler { false); ask2.add(request2); ask2.add(request3); - scheduler.allocate(id21, ask2, new ArrayList()); + scheduler.allocate(id21, ask2, new ArrayList(), null, null); // Third ask, queue2 requests 1 large List ask3 = new ArrayList(); ResourceRequest request4 = createResourceRequest(2 * minReqSize, ResourceRequest.ANY, 1, 1, true); ask3.add(request4); - scheduler.allocate(id22, ask3, new ArrayList()); + scheduler.allocate(id22, ask3, new ArrayList(), null, null); scheduler.update(); @@ -1369,7 +1369,7 @@ public class TestFairScheduler { // Complete container scheduler.allocate(attId, new ArrayList(), - Arrays.asList(containerId)); + Arrays.asList(containerId), null, null); assertEquals(1024, scheduler.getRootQueueMetrics().getAvailableMB()); // Schedule at opening @@ -1444,7 +1444,7 @@ public class TestFairScheduler { asks.add(createResourceRequest(1024, node3.getRackName(), 1, 1, true)); asks.add(createResourceRequest(1024, ResourceRequest.ANY, 1, 2, true)); - scheduler.allocate(appId, asks, new ArrayList()); + scheduler.allocate(appId, asks, new ArrayList(), null, null); // node 1 checks in scheduler.update(); @@ -1799,7 +1799,7 @@ public class TestFairScheduler { createResourceRequest(1024, node1.getHostName(), 1, 0, true), createResourceRequest(1024, "rack1", 1, 0, true), createResourceRequest(1024, ResourceRequest.ANY, 1, 1, true)); - scheduler.allocate(attId1, update, new ArrayList()); + scheduler.allocate(attId1, update, new ArrayList(), null, null); // then node2 should get the container scheduler.handle(node2UpdateEvent); @@ -1842,7 +1842,7 @@ public class TestFairScheduler { anyRequest = createResourceRequest(1024, ResourceRequest.ANY, 1, 1, false); scheduler.allocate(attId, Arrays.asList(rackRequest, anyRequest), - new ArrayList()); + new ArrayList(), null, null); scheduler.handle(nodeUpdateEvent); assertEquals(0, app.getReservedContainers().size()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java index 4fe344796dd..5af6585aff2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java @@ -186,7 +186,7 @@ public class TestFifoScheduler { ask.add(nodeLocal); ask.add(rackLocal); ask.add(any); - scheduler.allocate(appAttemptId, ask, new ArrayList()); + scheduler.allocate(appAttemptId, ask, new ArrayList(), null, null); NodeUpdateSchedulerEvent node0Update = new NodeUpdateSchedulerEvent(node0); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java index ece57b80333..985ccd69b52 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java @@ -501,7 +501,7 @@ public class TestContainerManagerSecurity { AllocateRequest allocateRequest = AllocateRequest.newInstance( BuilderUtils.newApplicationAttemptId(appID, 1), 0, 0F, ask, - new ArrayList()); + new ArrayList(), null); List allocatedContainers = scheduler.allocate(allocateRequest) .getAllocatedContainers();