From b2facc84a1b48b9dcbe0816e120778d2100b320e Mon Sep 17 00:00:00 2001 From: Surendra Singh Lilhore Date: Sat, 20 Jun 2020 19:55:23 +0530 Subject: [PATCH] YARN-9460. QueueACLsManager and ReservationsACLManager should not use instanceof checks. Contributed by Bilwa S T. --- .../resourcemanager/ResourceManager.java | 2 +- .../AbstractReservationSystem.java | 10 +- .../security/CapacityQueueACLsManager.java | 111 +++++++++++++++++ .../CapacityReservationsACLsManager.java | 46 +++++++ .../security/FairQueueACLsManager.java | 72 +++++++++++ .../security/FairReservationsACLsManager.java | 42 +++++++ .../security/GenericQueueACLsManager.java | 55 +++++++++ .../security/QueueACLsManager.java | 116 ++++-------------- .../security/ReservationsACLsManager.java | 44 ++----- .../security/package-info.java | 28 +++++ .../resourcemanager/TestClientRMTokens.java | 5 +- 11 files changed, 402 insertions(+), 129 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/CapacityQueueACLsManager.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/CapacityReservationsACLsManager.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/FairQueueACLsManager.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/FairReservationsACLsManager.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/GenericQueueACLsManager.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/package-info.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 48cbd8f6fc5..836a5ece80b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -438,7 +438,7 @@ public class ResourceManager extends CompositeService protected QueueACLsManager createQueueACLsManager(ResourceScheduler scheduler, Configuration conf) { - return new QueueACLsManager(scheduler, conf); + return QueueACLsManager.getQueueACLsManager(scheduler, conf); } @VisibleForTesting diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java index 5b8772c8541..d9e4be9e522 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java @@ -50,6 +50,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.security.CapacityReservationsACLsManager; +import org.apache.hadoop.yarn.server.resourcemanager.security.FairReservationsACLsManager; import org.apache.hadoop.yarn.server.resourcemanager.security.ReservationsACLsManager; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.UTCClock; @@ -173,7 +175,13 @@ public abstract class AbstractReservationSystem extends AbstractService YarnConfiguration.DEFAULT_YARN_RESERVATION_ACL_ENABLE) && conf.getBoolean(YarnConfiguration.YARN_ACL_ENABLE, YarnConfiguration.DEFAULT_YARN_ACL_ENABLE)) { - reservationsACLsManager = new ReservationsACLsManager(scheduler, conf); + if (scheduler instanceof CapacityScheduler) { + reservationsACLsManager = new CapacityReservationsACLsManager(scheduler, + conf); + } else if (scheduler instanceof FairScheduler) { + reservationsACLsManager = new FairReservationsACLsManager(scheduler, + conf); + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/CapacityQueueACLsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/CapacityQueueACLsManager.java new file mode 100644 index 00000000000..68a4530d616 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/CapacityQueueACLsManager.java @@ -0,0 +1,111 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.resourcemanager.security; + +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.records.QueueACL; +import org.apache.hadoop.yarn.security.AccessRequest; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This is the implementation of {@link QueueACLsManager} based on the + * {@link CapacityScheduler}. + */ +public class CapacityQueueACLsManager extends QueueACLsManager { + private static final Logger LOG = LoggerFactory + .getLogger(CapacityQueueACLsManager.class); + + public CapacityQueueACLsManager(ResourceScheduler scheduler, + Configuration conf) { + super(scheduler, conf); + } + + @Override + public boolean checkAccess(UserGroupInformation callerUGI, QueueACL acl, + RMApp app, String remoteAddress, List forwardedAddresses) { + if (!isACLsEnable) { + return true; + } + + CSQueue queue = ((CapacityScheduler) scheduler).getQueue(app.getQueue()); + if (queue == null) { + if (((CapacityScheduler) scheduler).isAmbiguous(app.getQueue())) { + LOG.error("Queue " + app.getQueue() + " is ambiguous for " + + app.getApplicationId()); + // if we cannot decide which queue to submit we should deny access + return false; + } + + // The application exists but the associated queue does not exist. + // This may be due to a queue that is not defined when the RM restarts. + // At this point we choose to log the fact and allow users to access + // and view the apps in a removed queue. This should only happen on + // application recovery. + LOG.error("Queue " + app.getQueue() + " does not exist for " + + app.getApplicationId()); + return true; + } + return authorizer.checkPermission( + new AccessRequest(queue.getPrivilegedEntity(), callerUGI, + SchedulerUtils.toAccessType(acl), app.getApplicationId().toString(), + app.getName(), remoteAddress, forwardedAddresses)); + + } + + @Override + public boolean checkAccess(UserGroupInformation callerUGI, QueueACL acl, + RMApp app, String remoteAddress, List forwardedAddresses, + String targetQueue) { + if (!isACLsEnable) { + return true; + } + + // Based on the discussion in YARN-5554 detail on why there are two + // versions: + // The access check inside these calls is currently scheduler dependent. + // This is due to the extra parameters needed for the CS case which are not + // in the version defined in the YarnScheduler interface. The second + // version is added for the moving the application case. The check has + // extra logging to distinguish between the queue not existing in the + // application move request case and the real access denied case. + CapacityScheduler cs = ((CapacityScheduler) scheduler); + CSQueue queue = cs.getQueue(targetQueue); + if (queue == null) { + LOG.warn("Target queue " + targetQueue + + (cs.isAmbiguous(targetQueue) ? " is ambiguous while trying to move " + : " does not exist while trying to move ") + + app.getApplicationId()); + return false; + } + return authorizer.checkPermission( + new AccessRequest(queue.getPrivilegedEntity(), callerUGI, + SchedulerUtils.toAccessType(acl), app.getApplicationId().toString(), + app.getName(), remoteAddress, forwardedAddresses)); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/CapacityReservationsACLsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/CapacityReservationsACLsManager.java new file mode 100644 index 00000000000..531d2a31533 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/CapacityReservationsACLsManager.java @@ -0,0 +1,46 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.hadoop.yarn.server.resourcemanager.security; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; + +/** + * This is the implementation of {@link ReservationsACLsManager} based on the + * {@link CapacityScheduler}. + */ +public class CapacityReservationsACLsManager extends ReservationsACLsManager { + + public CapacityReservationsACLsManager(ResourceScheduler scheduler, + Configuration conf) throws YarnException { + super(conf); + CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration( + conf); + + for (String planQueue : scheduler.getPlanQueues()) { + CSQueue queue = ((CapacityScheduler) scheduler).getQueue(planQueue); + reservationAcls.put(planQueue, + csConf.getReservationAcls(queue.getQueuePath())); + } + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/FairQueueACLsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/FairQueueACLsManager.java new file mode 100644 index 00000000000..688d46850b2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/FairQueueACLsManager.java @@ -0,0 +1,72 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.resourcemanager.security; + +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.records.QueueACL; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This is the implementation of {@link QueueACLsManager} based on the + * {@link FairScheduler}. + */ +public class FairQueueACLsManager extends QueueACLsManager { + private static final Logger LOG = LoggerFactory + .getLogger(FairQueueACLsManager.class); + + public FairQueueACLsManager(ResourceScheduler scheduler, Configuration conf) { + super(scheduler, conf); + } + + @Override + public boolean checkAccess(UserGroupInformation callerUGI, QueueACL acl, + RMApp app, String remoteAddress, List forwardedAddresses) { + if (!isACLsEnable) { + return true; + } + return scheduler.checkAccess(callerUGI, acl, app.getQueue()); + } + + @Override + public boolean checkAccess(UserGroupInformation callerUGI, QueueACL acl, + RMApp app, String remoteAddress, List forwardedAddresses, + String targetQueue) { + if (!isACLsEnable) { + return true; + } + + FSQueue queue = ((FairScheduler) scheduler).getQueueManager() + .getQueue(targetQueue); + if (queue == null) { + LOG.warn("Target queue " + targetQueue + + " does not exist while trying to move " + app.getApplicationId()); + return false; + } + return scheduler.checkAccess(callerUGI, acl, targetQueue); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/FairReservationsACLsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/FairReservationsACLsManager.java new file mode 100644 index 00000000000..09f147f89ea --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/FairReservationsACLsManager.java @@ -0,0 +1,42 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.hadoop.yarn.server.resourcemanager.security; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; + +/** + * This is the implementation of {@link ReservationsACLsManager} based on the + * {@link FairScheduler}. + */ +public class FairReservationsACLsManager extends ReservationsACLsManager { + + public FairReservationsACLsManager(ResourceScheduler scheduler, + Configuration conf) throws YarnException { + super(conf); + AllocationConfiguration aConf = ((FairScheduler) scheduler) + .getAllocationConfiguration(); + for (String planQueue : scheduler.getPlanQueues()) { + reservationAcls.put(planQueue, aConf.getReservationAcls(planQueue)); + } + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/GenericQueueACLsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/GenericQueueACLsManager.java new file mode 100644 index 00000000000..5f3559c65e8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/GenericQueueACLsManager.java @@ -0,0 +1,55 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.hadoop.yarn.server.resourcemanager.security; + +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.records.QueueACL; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This is the generic implementation of {@link QueueACLsManager}. + */ +public class GenericQueueACLsManager extends QueueACLsManager { + + private static final Logger LOG = LoggerFactory + .getLogger(GenericQueueACLsManager.class); + + public GenericQueueACLsManager(ResourceScheduler scheduler, + Configuration conf) { + super(scheduler, conf); + } + + @Override + public boolean checkAccess(UserGroupInformation callerUGI, QueueACL acl, + RMApp app, String remoteAddress, List forwardedAddresses) { + return scheduler.checkAccess(callerUGI, acl, app.getQueue()); + } + + @Override + public boolean checkAccess(UserGroupInformation callerUGI, QueueACL acl, + RMApp app, String remoteAddress, List forwardedAddresses, + String targetQueue) { + return scheduler.checkAccess(callerUGI, acl, targetQueue); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java index f13608c0ac6..290ae7c5d3a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java @@ -19,35 +19,26 @@ package org.apache.hadoop.yarn.server.resourcemanager.security; import com.google.common.annotations.VisibleForTesting; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.security.AccessRequest; import org.apache.hadoop.yarn.security.YarnAuthorizationProvider; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; - import java.util.List; -public class QueueACLsManager { +@SuppressWarnings("checkstyle:visibilitymodifier") +public abstract class QueueACLsManager { - private static final Logger LOG = - LoggerFactory.getLogger(QueueACLsManager.class); - - private ResourceScheduler scheduler; - private boolean isACLsEnable; - private YarnAuthorizationProvider authorizer; + ResourceScheduler scheduler; + boolean isACLsEnable; + YarnAuthorizationProvider authorizer; @VisibleForTesting - public QueueACLsManager() { + public QueueACLsManager(Configuration conf) { this(null, new Configuration()); } @@ -58,41 +49,27 @@ public class QueueACLsManager { this.authorizer = YarnAuthorizationProvider.getInstance(conf); } - public boolean checkAccess(UserGroupInformation callerUGI, QueueACL acl, - RMApp app, String remoteAddress, List forwardedAddresses) { - if (!isACLsEnable) { - return true; - } - + /** + * Get queue acl manager corresponding to the scheduler. + * @param scheduler the scheduler for which the queue acl manager is required + * @param conf + * @return {@link QueueACLsManager} + */ + public static QueueACLsManager getQueueACLsManager( + ResourceScheduler scheduler, Configuration conf) { if (scheduler instanceof CapacityScheduler) { - CSQueue queue = ((CapacityScheduler) scheduler).getQueue(app.getQueue()); - if (queue == null) { - if (((CapacityScheduler) scheduler).isAmbiguous(app.getQueue())) { - LOG.error("Queue " + app.getQueue() + " is ambiguous for " - + app.getApplicationId()); - //if we cannot decide which queue to submit we should deny access - return false; - } - - // The application exists but the associated queue does not exist. - // This may be due to a queue that is not defined when the RM restarts. - // At this point we choose to log the fact and allow users to access - // and view the apps in a removed queue. This should only happen on - // application recovery. - LOG.error("Queue " + app.getQueue() + " does not exist for " + app - .getApplicationId()); - return true; - } - return authorizer.checkPermission( - new AccessRequest(queue.getPrivilegedEntity(), callerUGI, - SchedulerUtils.toAccessType(acl), - app.getApplicationId().toString(), app.getName(), - remoteAddress, forwardedAddresses)); + return new CapacityQueueACLsManager(scheduler, conf); + } else if (scheduler instanceof FairScheduler) { + return new FairQueueACLsManager(scheduler, conf); } else { - return scheduler.checkAccess(callerUGI, acl, app.getQueue()); + return new GenericQueueACLsManager(scheduler, conf); } } + public abstract boolean checkAccess(UserGroupInformation callerUGI, + QueueACL acl, RMApp app, String remoteAddress, + List forwardedAddresses); + /** * Check access to a targetQueue in the case of a move of an application. * The application cannot contain the destination queue since it has not @@ -107,50 +84,7 @@ public class QueueACLsManager { * @return true: if submission is allowed and queue exists, * false: in all other cases (also non existing target queue) */ - public boolean checkAccess(UserGroupInformation callerUGI, QueueACL acl, - RMApp app, String remoteAddress, List forwardedAddresses, - String targetQueue) { - if (!isACLsEnable) { - return true; - } - - // Based on the discussion in YARN-5554 detail on why there are two - // versions: - // The access check inside these calls is currently scheduler dependent. - // This is due to the extra parameters needed for the CS case which are not - // in the version defined in the YarnScheduler interface. The second - // version is added for the moving the application case. The check has - // extra logging to distinguish between the queue not existing in the - // application move request case and the real access denied case. - if (scheduler instanceof CapacityScheduler) { - CapacityScheduler cs = ((CapacityScheduler) scheduler); - CSQueue queue = cs.getQueue(targetQueue); - if (queue == null) { - LOG.warn("Target queue " + targetQueue - + (cs.isAmbiguous(targetQueue) ? - " is ambiguous while trying to move " : - " does not exist while trying to move ") - + app.getApplicationId()); - return false; - } - return authorizer.checkPermission( - new AccessRequest(queue.getPrivilegedEntity(), callerUGI, - SchedulerUtils.toAccessType(acl), - app.getApplicationId().toString(), app.getName(), - remoteAddress, forwardedAddresses)); - } else if (scheduler instanceof FairScheduler) { - FSQueue queue = ((FairScheduler) scheduler).getQueueManager(). - getQueue(targetQueue); - if (queue == null) { - LOG.warn("Target queue " + targetQueue - + " does not exist while trying to move " - + app.getApplicationId()); - return false; - } - return scheduler.checkAccess(callerUGI, acl, targetQueue); - } else { - // Any other scheduler just try - return scheduler.checkAccess(callerUGI, acl, targetQueue); - } - } + public abstract boolean checkAccess(UserGroupInformation callerUGI, + QueueACL acl, RMApp app, String remoteAddress, + List forwardedAddresses, String targetQueue); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ReservationsACLsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ReservationsACLsManager.java index be2be184f36..6fc9953f79a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ReservationsACLsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ReservationsACLsManager.java @@ -24,50 +24,26 @@ import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.yarn.api.records.ReservationACL; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfiguration; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; - import java.util.HashMap; import java.util.Map; /** * The {@link ReservationsACLsManager} is used to check a specified user's * permissons to perform a reservation operation on the - * {@link CapacityScheduler} and the {@link FairScheduler}. * {@link ReservationACL}s are used to specify reservation operations. */ -public class ReservationsACLsManager { +@SuppressWarnings("checkstyle:visibilitymodifier") +public abstract class ReservationsACLsManager { private boolean isReservationACLsEnable; - private Map> reservationAcls - = new HashMap<>(); + Map> reservationAcls = + new HashMap<>(); - public ReservationsACLsManager(ResourceScheduler scheduler, - Configuration conf) throws YarnException { - this.isReservationACLsEnable = - conf.getBoolean(YarnConfiguration.YARN_RESERVATION_ACL_ENABLE, - YarnConfiguration.DEFAULT_YARN_RESERVATION_ACL_ENABLE) && - conf.getBoolean(YarnConfiguration.YARN_ACL_ENABLE, - YarnConfiguration.DEFAULT_YARN_ACL_ENABLE); - if (scheduler instanceof CapacityScheduler) { - CapacitySchedulerConfiguration csConf = new - CapacitySchedulerConfiguration(conf); - - for (String planQueue : scheduler.getPlanQueues()) { - CSQueue queue = ((CapacityScheduler) scheduler).getQueue(planQueue); - reservationAcls.put(planQueue, csConf.getReservationAcls(queue - .getQueuePath())); - } - } else if (scheduler instanceof FairScheduler) { - AllocationConfiguration aConf = ((FairScheduler) scheduler) - .getAllocationConfiguration(); - for (String planQueue : scheduler.getPlanQueues()) { - reservationAcls.put(planQueue, aConf.getReservationAcls(planQueue)); - } - } + public ReservationsACLsManager(Configuration conf) throws YarnException { + this.isReservationACLsEnable = conf.getBoolean( + YarnConfiguration.YARN_RESERVATION_ACL_ENABLE, + YarnConfiguration.DEFAULT_YARN_RESERVATION_ACL_ENABLE) + && conf.getBoolean(YarnConfiguration.YARN_ACL_ENABLE, + YarnConfiguration.DEFAULT_YARN_ACL_ENABLE); } public boolean checkAccess(UserGroupInformation callerUGI, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/package-info.java new file mode 100644 index 00000000000..dcc2d871031 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/package-info.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Package org.apache.hadoop.yarn.server.resourcemanager.security + * contains classes related to security. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +package org.apache.hadoop.yarn.server.resourcemanager.security; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java index e700bfd1e87..50afced670e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java @@ -544,8 +544,9 @@ public class TestClientRMTokens { ResourceScheduler scheduler, RMDelegationTokenSecretManager rmDTSecretManager) { super(mock(RMContext.class), scheduler, mock(RMAppManager.class), - new ApplicationACLsManager(conf), new QueueACLsManager(scheduler, - conf), rmDTSecretManager); + new ApplicationACLsManager(conf), + QueueACLsManager.getQueueACLsManager(scheduler, conf), + rmDTSecretManager); } // Use a random port unless explicitly specified.